nik/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-08-31 07:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Show main and sub stream usage separately in Storage Metrics (#24015)

Browse Source 
* backend

* frontend

* docs

* test

* report null instead of 0 for a stream with no cached bandwidth sample
This commit is contained in:
Josh Hawkins 2026-08-18 08:25:38 -05:00
parent dfe6428111
commit 91a93167d2
7 changed files with 397 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/docs/configuration/record.md
View File

@ -421,7 +421,7 @@ As a general rule, features that read recordings prefer the main stream and fall
| Audio extraction (e.g., transcription) | Main preferred, sub fallback |
| Motion search | Main only |
| Review timeline motion data | Main only |
| Storage usage statistics | Both streams counted |
| Storage usage statistics | Both streams counted, and listed separately per camera |
This table covers only features that read recordings from disk. Tracked object snapshots and thumbnails (the images shown in Explore and sent with notifications, and the images submitted to Frigate+ from a tracked object) are captured live from the `detect` stream as the object is tracked, never from recordings, so sub stream recording does not affect them.

48
frigate/storage.py
View File

@ -88,7 +88,7 @@ class StorageMaintainer(threading.Thread):
# type and sum the rates; mixing streams would average small
# sub segments against large main segments and underestimate
# the true write rate
bandwidth = 0.0
bandwidth_by_stream: dict[str, float] = {}
for stream_type in (STREAM_TYPE_MAIN, STREAM_TYPE_SUB):
avg_bw = self._recent_stream_bandwidth(camera, stream_type, 100)
if avg_bw is None:
@ -99,17 +99,27 @@ class StorageMaintainer(threading.Thread):
camera, stream_type, 1000
)
if avg_bw is not None:
bandwidth += round(avg_bw * 3600, 2)
bandwidth_by_stream[stream_type] = round(avg_bw * 3600, 2)
bandwidth = round(bandwidth, 2)
bandwidth = round(sum(bandwidth_by_stream.values()), 2)
if bandwidth > MAX_CALCULATED_BANDWIDTH:
logger.warning(
f"{camera} has a bandwidth of {bandwidth} MB/hr which exceeds the expected maximum. This typically indicates an issue with the cameras recordings."
)
# scale each stream so the per stream values still sum to
# the clamped total the UI displays alongside them
scale = MAX_CALCULATED_BANDWIDTH / bandwidth
bandwidth_by_stream = {
stream_type: round(value * scale, 2)
for stream_type, value in bandwidth_by_stream.items()
}
bandwidth = MAX_CALCULATED_BANDWIDTH
self.camera_storage_stats[camera]["bandwidth"] = bandwidth
self.camera_storage_stats[camera]["bandwidth_by_stream"] = (
bandwidth_by_stream
)
logger.debug(f"{camera} has a bandwidth of {bandwidth} MiB/hr.")
def calculate_camera_usages(self) -> dict[str, dict]:
@ -121,20 +131,42 @@ class StorageMaintainer(threading.Thread):
if camera.startswith(REPLAY_CAMERA_PREFIX):
continue
camera_storage = (
Recordings.select(fn.SUM(Recordings.segment_size))
.where(Recordings.camera == camera, Recordings.segment_size != 0)
.scalar()
stream_usages = {
row["stream_type"]: row["usage"] or 0
for row in (
Recordings.select(
Recordings.stream_type,
fn.SUM(Recordings.segment_size).alias("usage"),
)
.where(Recordings.camera == camera, Recordings.segment_size != 0)
.group_by(Recordings.stream_type)
.dicts()
)
}
stream_bandwidths = self.camera_storage_stats.get(camera, {}).get(
"bandwidth_by_stream", {}
)
camera_key = (
getattr(self.config.cameras[camera], "friendly_name", None) or camera
)
usages[camera_key] = {
"usage": camera_storage,
"usage": sum(stream_usages.values()),
"bandwidth": self.camera_storage_stats.get(camera, {}).get(
"bandwidth", 0
),
# only streams with segments on disk are reported, so a camera
# keeps its sub entry until sub retention expires those segments.
# bandwidth is null rather than 0 when the cache holds no sample
# for the stream, since 0 would claim it writes nothing
"streams": {
stream_type: {
"usage": stream_usages[stream_type],
"bandwidth": stream_bandwidths.get(stream_type),
}
for stream_type in (STREAM_TYPE_MAIN, STREAM_TYPE_SUB)
if stream_usages.get(stream_type)
},
}
return usages

175
frigate/test/test_storage.py
View File

@ -11,8 +11,9 @@ from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqliteq import SqliteQueueDatabase
from frigate.config import FrigateConfig
from frigate.const import STREAM_TYPE_MAIN, STREAM_TYPE_SUB
from frigate.models import Event, Recordings
from frigate.storage import StorageMaintainer
from frigate.storage import MAX_CALCULATED_BANDWIDTH, StorageMaintainer
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
@ -114,8 +115,16 @@ class TestHttp(unittest.TestCase):
)
storage.calculate_camera_bandwidth()
assert storage.camera_storage_stats == {
"front_door": {"bandwidth": 1440, "needs_refresh": True},
"back_door": {"bandwidth": 2880, "needs_refresh": True},
"front_door": {
"bandwidth": 1440,
"bandwidth_by_stream": {STREAM_TYPE_MAIN: 1440},
"needs_refresh": True,
},
"back_door": {
"bandwidth": 2880,
"bandwidth_by_stream": {STREAM_TYPE_MAIN: 2880},
"needs_refresh": True,
},
}
def test_segment_calculations_with_zero_segments(self):
@ -136,7 +145,11 @@ class TestHttp(unittest.TestCase):
)
storage.calculate_camera_bandwidth()
assert storage.camera_storage_stats == {
"front_door": {"bandwidth": 0, "needs_refresh": True},
"front_door": {
"bandwidth": 0,
"bandwidth_by_stream": {},
"needs_refresh": True,
},
}
def test_segment_calculations_with_recent_zero_segments(self):
@ -171,9 +184,159 @@ class TestHttp(unittest.TestCase):
storage.calculate_camera_bandwidth()
assert storage.camera_storage_stats == {
"front_door": {"bandwidth": 1440, "needs_refresh": True},
"front_door": {
"bandwidth": 1440,
"bandwidth_by_stream": {STREAM_TYPE_MAIN: 1440},
"needs_refresh": True,
},
}
def test_camera_usages_split_by_stream_type(self):
"""Usage and bandwidth are reported per stream type."""
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock())
time_keep = datetime.datetime.now().timestamp()
_insert_mock_recording(
"1234567.frontdoor",
os.path.join(self.test_dir, "main.tmp"),
time_keep,
time_keep + 10,
seg_size=20,
seg_dur=10,
stream_type=STREAM_TYPE_MAIN,
)
_insert_mock_recording(
"1234568.frontdoor",
os.path.join(self.test_dir, "sub.tmp"),
time_keep,
time_keep + 10,
seg_size=2,
seg_dur=10,
stream_type=STREAM_TYPE_SUB,
)
storage.calculate_camera_bandwidth()
usages = storage.calculate_camera_usages()
assert usages["front_door"]["usage"] == 22
assert usages["front_door"]["bandwidth"] == 7920
assert usages["front_door"]["streams"] == {
STREAM_TYPE_MAIN: {"usage": 20, "bandwidth": 7200},
STREAM_TYPE_SUB: {"usage": 2, "bandwidth": 720},
}
def test_camera_usages_omits_streams_without_segments(self):
"""A camera with no sub segments reports no sub entry."""
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock())
time_keep = datetime.datetime.now().timestamp()
_insert_mock_recording(
"1234567.frontdoor",
os.path.join(self.test_dir, "main.tmp"),
time_keep,
time_keep + 10,
seg_size=20,
seg_dur=10,
)
storage.calculate_camera_bandwidth()
usages = storage.calculate_camera_usages()
assert usages["front_door"]["usage"] == 20
assert usages["front_door"]["streams"] == {
STREAM_TYPE_MAIN: {"usage": 20, "bandwidth": 7200},
}
def test_camera_bandwidth_clamp_scales_stream_values(self):
"""Clamping the total keeps the per stream values summing to it."""
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock())
time_keep = datetime.datetime.now().timestamp()
_insert_mock_recording(
"1234567.frontdoor",
os.path.join(self.test_dir, "main.tmp"),
time_keep,
time_keep + 10,
seg_size=40,
seg_dur=10,
stream_type=STREAM_TYPE_MAIN,
)
_insert_mock_recording(
"1234568.frontdoor",
os.path.join(self.test_dir, "sub.tmp"),
time_keep,
time_keep + 10,
seg_size=4,
seg_dur=10,
stream_type=STREAM_TYPE_SUB,
)
storage.calculate_camera_bandwidth()
stats = storage.camera_storage_stats["front_door"]
assert stats["bandwidth"] == MAX_CALCULATED_BANDWIDTH
assert (
round(sum(stats["bandwidth_by_stream"].values()), 2)
== MAX_CALCULATED_BANDWIDTH
)
def test_stream_bandwidth_is_none_without_a_cached_sample(self):
"""A stream that appears after the bandwidth cache freezes has no estimate.
Sub stream recording can be toggled on at runtime, so the cache can hold
a main-only sample while sub segments are already landing on disk.
Reporting 0 there would claim the sub stream costs nothing.
"""
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock())
time_keep = datetime.datetime.now().timestamp()
for i in range(60):
_insert_mock_recording(
f"main_{i}.frontdoor",
os.path.join(self.test_dir, f"main_{i}.tmp"),
time_keep + i * 10,
time_keep + i * 10 + 10,
seg_size=20,
seg_dur=10,
)
# 50 or more segments flips needs_refresh off, freezing the cache
storage.calculate_camera_bandwidth()
assert storage.camera_storage_stats["front_door"]["needs_refresh"] is False
for i in range(60):
_insert_mock_recording(
f"sub_{i}.frontdoor",
os.path.join(self.test_dir, f"sub_{i}.tmp"),
time_keep + 5000 + i * 10,
time_keep + 5000 + i * 10 + 10,
seg_size=2,
seg_dur=10,
stream_type=STREAM_TYPE_SUB,
)
storage.calculate_camera_bandwidth()
streams = storage.calculate_camera_usages()["front_door"]["streams"]
assert streams[STREAM_TYPE_SUB]["usage"] == 120
assert streams[STREAM_TYPE_SUB]["bandwidth"] is None
assert streams[STREAM_TYPE_MAIN]["bandwidth"] == 7200
def test_camera_usages_with_no_recordings(self):
"""A camera with no segments reports zero usage and no streams."""
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock())
storage.calculate_camera_bandwidth()
usages = storage.calculate_camera_usages()
assert usages["front_door"]["usage"] == 0
assert usages["front_door"]["streams"] == {}
def test_storage_cleanup(self):
"""Ensure that all recordings are cleaned up when necessary."""
config = FrigateConfig(**self.minimal_config)
@ -332,6 +495,7 @@ def _insert_mock_recording(
camera="front_door",
seg_size=8,
seg_dur=10,
stream_type=STREAM_TYPE_MAIN,
) -> Event:
"""Inserts a basic recording model with a given id."""
# we must open the file so storage maintainer will delete it
@ -348,4 +512,5 @@ def _insert_mock_recording(
motion=True,
objects=True,
segment_size=seg_size,
stream_type=stream_type,
).execute()

4
web/public/locales/en/views/system.json
View File

@ -148,6 +148,10 @@
"unused": {
"title": "Unused",
"tips": "This value may not accurately represent the free space available to Frigate if you have other files stored on your drive beyond Frigate's recordings. Frigate does not track storage usage outside of its recordings."
},
"subStream": {
"information": "Sub Stream Storage Information",
"tips": "Cameras that record a sub stream show their Original and Low quality usage separately, using the same names as the quality options in recording playback. Low quality recordings stay on disk until their retention period expires, so this breakdown can still appear for a camera after sub stream recording has been turned off."
}
}
},

243
web/src/components/graph/CombinedStorageGraph.tsx
View File

@ -16,24 +16,28 @@ import {
PopoverTrigger,
} from "@/components/ui/popover";
import { getUnitSize } from "@/utils/storageUtil";
import { CameraStorage, StreamStorage } from "@/types/stats";
import { CiCircleAlert } from "react-icons/ci";
import { useTranslation } from "react-i18next";
type CameraStorage = {
[key: string]: {
bandwidth: number;
usage: number;
usage_percent: number;
};
};
type TotalStorage = {
used: number;
camera: number;
total: number;
};
type StorageRow = {
name: string;
usage: number;
bandwidth: number;
color: string;
streams?: {
main?: StreamStorage;
sub?: StreamStorage;
};
};
type CombinedStorageGraphProps = {
graphId: string;
cameraStorage: CameraStorage;
@ -44,46 +48,59 @@ export function CombinedStorageGraph({
cameraStorage,
totalStorage,
}: CombinedStorageGraphProps) {
const { t } = useTranslation(["views/system"]);
const { t } = useTranslation(["views/system", "components/player"]);
const { theme, systemTheme } = useTheme();
const isDark = (systemTheme || theme) == "dark";
const entities = Object.keys(cameraStorage);
const colors = generateColors(entities.length);
const rows: StorageRow[] = useMemo(() => {
const entities = Object.keys(cameraStorage);
const colors = generateColors(entities.length);
const series = entities.map((entity, index) => ({
name: entity,
data: [(cameraStorage[entity].usage / totalStorage.total) * 100],
usage: cameraStorage[entity].usage,
bandwidth: cameraStorage[entity].bandwidth,
color: colors[index], // Assign the corresponding color
}));
return [
...entities.map((entity, index) => ({
name: entity,
usage: cameraStorage[entity].usage ?? 0,
bandwidth: cameraStorage[entity].bandwidth,
color: colors[index],
streams: cameraStorage[entity].streams,
})),
{
name: "Other",
usage: totalStorage.used - totalStorage.camera,
bandwidth: 0,
color: isDark ? "#606060" : "#D5D5D5",
},
{
name: "Unused",
usage: totalStorage.total - totalStorage.used,
bandwidth: 0,
color: isDark ? "#404040" : "#E5E5E5",
},
];
}, [cameraStorage, totalStorage, isDark]);
// Add the unused percentage to the series
series.push({
name: "Other",
data: [
((totalStorage.used - totalStorage.camera) / totalStorage.total) * 100,
],
usage: totalStorage.used - totalStorage.camera,
bandwidth: 0,
color: (systemTheme || theme) == "dark" ? "#606060" : "#D5D5D5",
});
series.push({
name: "Unused",
data: [
((totalStorage.total - totalStorage.used) / totalStorage.total) * 100,
],
usage: totalStorage.total - totalStorage.used,
bandwidth: 0,
color: (systemTheme || theme) == "dark" ? "#404040" : "#E5E5E5",
});
const series = useMemo(
() =>
rows.map((row) => ({
name: row.name,
data: [(row.usage / totalStorage.total) * 100],
usage: row.usage,
color: row.color,
})),
[rows, totalStorage.total],
);
const hasSubStorage = useMemo(
() => rows.some((row) => row.streams?.sub),
[rows],
);
const options = useMemo(() => {
return {
chart: {
id: graphId,
background: (systemTheme || theme) == "dark" ? "#404040" : "#E5E5E5",
background: isDark ? "#404040" : "#E5E5E5",
selection: {
enabled: false,
},
@ -164,7 +181,7 @@ export function CombinedStorageGraph({
max: 100,
},
} as ApexCharts.ApexOptions;
}, [graphId, systemTheme, theme, series]);
}, [graphId, isDark, systemTheme, theme, series]);
useEffect(() => {
ApexCharts.exec(graphId, "updateOptions", options, true, true);
@ -185,6 +202,37 @@ export function CombinedStorageGraph({
[t],
);
const getStreamSplit = useCallback(
(row: StorageRow, field: keyof StreamStorage) => {
const mainValue = row.streams?.main?.[field];
const subValue = row.streams?.sub?.[field];
// omit the split entirely when either side is unknown, rather than
// showing a zero that would read as "this stream costs nothing"
if (mainValue == null || subValue == null) {
return null;
}
return (
<div className="text-xs text-primary-variant">
<div>
<span className="text-muted-foreground">
{t("quality.main", { ns: "components/player" })}
</span>{" "}
{getUnitSize(mainValue)}
</div>
<div>
<span className="text-muted-foreground">
{t("quality.sub", { ns: "components/player" })}
</span>{" "}
{getUnitSize(subValue)}
</div>
</div>
);
},
[t],
);
return (
<div className="flex w-full flex-col gap-2.5">
<div className="flex w-full items-center justify-between gap-1">
@ -206,7 +254,35 @@ export function CombinedStorageGraph({
<TableHeader>
<TableRow>
<TableHead>{t("storage.cameraStorage.camera")}</TableHead>
<TableHead>{t("storage.cameraStorage.storageUsed")}</TableHead>
<TableHead>
<div className="flex flex-row items-center gap-1">
{t("storage.cameraStorage.storageUsed")}
{hasSubStorage && (
<Popover>
<PopoverTrigger asChild>
<button
className="focus:outline-none"
aria-label={t(
"storage.cameraStorage.subStream.information",
)}
>
<CiCircleAlert
className="size-5"
aria-label={t(
"storage.cameraStorage.subStream.information",
)}
/>
</button>
</PopoverTrigger>
<PopoverContent className="w-80">
<div className="space-y-2">
{t("storage.cameraStorage.subStream.tips")}
</div>
</PopoverContent>
</Popover>
)}
</div>
</TableHead>
<TableHead>
{t("storage.cameraStorage.percentageOfTotalUsed")}
</TableHead>
@ -214,49 +290,56 @@ export function CombinedStorageGraph({
</TableRow>
</TableHeader>
<TableBody>
{series.map((item) => (
<TableRow key={item.name}>
<TableCell className="flex flex-row items-center gap-2 font-medium smart-capitalize">
{" "}
<div
className="size-3 rounded-md"
style={{ backgroundColor: item.color }}
></div>
{getItemTitle(item.name)}
{(item.name === "Unused" || item.name == "Other") && (
<Popover>
<PopoverTrigger asChild>
<button
className="focus:outline-none"
aria-label={t(
"storage.cameraStorage.unusedStorageInformation",
)}
>
<CiCircleAlert
className="size-5"
{rows.map((row) => {
const isAggregate = row.name == "Unused" || row.name == "Other";
return (
<TableRow key={row.name}>
<TableCell className="flex flex-row items-center gap-2 font-medium smart-capitalize">
<div
className="size-3 rounded-md"
style={{ backgroundColor: row.color }}
></div>
{getItemTitle(row.name)}
{isAggregate && (
<Popover>
<PopoverTrigger asChild>
<button
className="focus:outline-none"
aria-label={t(
"storage.cameraStorage.unusedStorageInformation",
)}
/>
</button>
</PopoverTrigger>
<PopoverContent className="w-80">
<div className="space-y-2">
{t("storage.cameraStorage.unused.tips")}
</div>
</PopoverContent>
</Popover>
)}
</TableCell>
<TableCell>{getUnitSize(item.usage ?? 0)}</TableCell>
<TableCell>{item.data[0].toFixed(2)}%</TableCell>
<TableCell>
{item.name === "Unused" || item.name == "Other"
? "—"
: `${getUnitSize(item.bandwidth)} / hour`}
</TableCell>
</TableRow>
))}
>
<CiCircleAlert
className="size-5"
aria-label={t(
"storage.cameraStorage.unusedStorageInformation",
)}
/>
</button>
</PopoverTrigger>
<PopoverContent className="w-80">
<div className="space-y-2">
{t("storage.cameraStorage.unused.tips")}
</div>
</PopoverContent>
</Popover>
)}
</TableCell>
<TableCell>
{getUnitSize(row.usage)}
{getStreamSplit(row, "usage")}
</TableCell>
<TableCell>
{((row.usage / totalStorage.total) * 100).toFixed(2)}%
</TableCell>
<TableCell>
{isAggregate ? "—" : `${getUnitSize(row.bandwidth)} / hour`}
{getStreamSplit(row, "bandwidth")}
</TableCell>
</TableRow>
);
})}
</TableBody>
</Table>
</div>

18
web/src/types/stats.ts
View File

@ -101,6 +101,24 @@ export type StorageStats = {
shm_frame_count?: number;
};
export type StreamStorage = {
usage: number;
bandwidth: number | null;
};
export type CameraStorage = {
[camera: string]: {
bandwidth: number;
usage: number;
usage_percent: number;
// absent for stream types with no segments on disk
streams?: {
main?: StreamStorage;
sub?: StreamStorage;
};
};
};
export type PotentialProblem = {
text: string;
color: string;

10
web/src/views/system/StorageMetrics.tsx
View File

@ -1,6 +1,6 @@
import { CombinedStorageGraph } from "@/components/graph/CombinedStorageGraph";
import { StorageGraph } from "@/components/graph/StorageGraph";
import { FrigateStats } from "@/types/stats";
import { CameraStorage, FrigateStats } from "@/types/stats";
import { useEffect, useMemo } from "react";
import {
Popover,
@ -24,14 +24,6 @@ import { LuExternalLink } from "react-icons/lu";
import { FaExclamationTriangle } from "react-icons/fa";
import ActivityIndicator from "@/components/indicators/activity-indicator";
type CameraStorage = {
[key: string]: {
bandwidth: number;
usage: number;
usage_percent: number;
};
};
type StorageMetricsProps = {
setLastUpdated: (last: number) => void;
};