Skip to content

Commit 78213f5

Browse files
TommyUngerTommyUnger
committed
snapshot diff tool and system metrics
1 parent b766a6b commit 78213f5

File tree

2 files changed

+216
-1
lines changed

2 files changed

+216
-1
lines changed

‎notebooks/Snapshot Diff Tool.ipynb

+215
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Snapshot Policy + Snapshot Diff Tool\n",
8+
"This tool will create an every minute snapshot policy. This will allow keeping track of every file create and/or modify. Use cases that this tool could help support include:\n",
9+
"* Watching for file changes to kick off a virus scan\n",
10+
"* Watching for the unexpected creation of files\n",
11+
"* Supporting a full index of the file system for search, find, etc"
12+
]
13+
},
14+
{
15+
"cell_type": "code",
16+
"execution_count": 100,
17+
"metadata": {},
18+
"outputs": [],
19+
"source": [
20+
"import re\n",
21+
"import time\n",
22+
"import pprint\n",
23+
"from qumulo.rest_client import RestClient\n",
24+
"\n",
25+
"rc = RestClient(\"qumulo.test\", 8000)\n",
26+
"rc.login(\"admin\", \"*****\");"
27+
]
28+
},
29+
{
30+
"cell_type": "markdown",
31+
"metadata": {},
32+
"source": [
33+
"## Create Snapshot Policy - Every Minute at Root\n",
34+
"Saves 10 minutes of snapshots as a small buffer."
35+
]
36+
},
37+
{
38+
"cell_type": "code",
39+
"execution_count": 101,
40+
"metadata": {},
41+
"outputs": [],
42+
"source": [
43+
"def create_policy_for_diff(policy_name=\"EveryMinuteForDiffs\", minutes=10):\n",
44+
" policy = rc.snapshot.create_policy(\n",
45+
" name = policy_name,\n",
46+
" schedule_info = {\"creation_schedule\":{\n",
47+
" \"frequency\":\"SCHEDULE_HOURLY_OR_LESS\",\n",
48+
" \"fire_every\":1,\n",
49+
" \"fire_every_interval\":\"FIRE_IN_MINUTES\",\n",
50+
" \"window_start_hour\":0,\n",
51+
" \"window_start_minute\":0,\n",
52+
" \"window_end_hour\":23,\n",
53+
" \"window_end_minute\":59,\n",
54+
" \"on_days\":[\"MON\",\"TUE\",\"WED\",\"THU\",\"FRI\",\"SAT\",\"SUN\"],\n",
55+
" \"timezone\":\"America/Los_Angeles\"\n",
56+
" },\n",
57+
" \"expiration_time_to_live\":\"%sminutes\" % minutes\n",
58+
" }\n",
59+
" )\n",
60+
" print(\"Created policy: %s expires after %s\" % (policy['name'], policy['schedules'][0]['expiration_time_to_live']))"
61+
]
62+
},
63+
{
64+
"cell_type": "markdown",
65+
"metadata": {},
66+
"source": [
67+
"## Diff all snapshots in the every minute policy"
68+
]
69+
},
70+
{
71+
"cell_type": "code",
72+
"execution_count": 109,
73+
"metadata": {},
74+
"outputs": [],
75+
"source": [
76+
"def diff_snaps(rc, policy_name):\n",
77+
" snap_count = 2 # set up for the 1st loop\n",
78+
" paths = []\n",
79+
" while snap_count >= 2:\n",
80+
" all_snaps = rc.snapshot.list_snapshot_statuses()['entries']\n",
81+
" short_list = filter(lambda s: s['name'] == policy_name, all_snaps)\n",
82+
" snaps = sorted(short_list, key=lambda s: s['id'])\n",
83+
" if len(snaps) < 2:\n",
84+
" break\n",
85+
" print(\"Diff times: %s -> %s\" % (snaps[0]['timestamp'][0:19], snaps[1]['timestamp'][0:19]))\n",
86+
" diff = rc.snapshot.get_all_snapshot_tree_diff(snaps[1]['id'], snaps[0]['id'])\n",
87+
" for d in diff:\n",
88+
" for e in d['entries']:\n",
89+
" if e['path'][-1] == \"/\":\n",
90+
" continue # it's a directory\n",
91+
" sz = None\n",
92+
" owner = None\n",
93+
" try:\n",
94+
" dets = rc.fs.get_file_attr(e['path'])\n",
95+
" sz = dets['size']\n",
96+
" owner = dets['owner_details']['id_value']\n",
97+
" except:\n",
98+
" pass\n",
99+
" if e['op'] == 'DELETE' and sz is not None:\n",
100+
" continue # don't add deletes for existing files\n",
101+
" paths.append({'op': e['op'],\n",
102+
" 'path': e['path'], \n",
103+
" 'size': sz, \n",
104+
" 'owner': owner,\n",
105+
" 'snapshot_id': snaps[1]['id']})\n",
106+
" # delete the oldest snapshot\n",
107+
" rc.snapshot.delete_snapshot(snaps[0]['id'])\n",
108+
" snap_count = len(snaps) - 1\n",
109+
" return paths\n"
110+
]
111+
},
112+
{
113+
"cell_type": "code",
114+
"execution_count": 103,
115+
"metadata": {},
116+
"outputs": [
117+
{
118+
"name": "stdout",
119+
"output_type": "stream",
120+
"text": [
121+
"Created policy: EveryMinuteForDiffs expires after 10minutes\n"
122+
]
123+
}
124+
],
125+
"source": [
126+
"create_policy_for_diff('EveryMinuteForDiffs')"
127+
]
128+
},
129+
{
130+
"cell_type": "code",
131+
"execution_count": 110,
132+
"metadata": {},
133+
"outputs": [
134+
{
135+
"name": "stdout",
136+
"output_type": "stream",
137+
"text": [
138+
"Diff times: 2020-04-28T16:43:23 -> 2020-04-28T16:44:23\n",
139+
"Diff times: 2020-04-28T16:44:23 -> 2020-04-28T16:45:23\n",
140+
"Diff times: 2020-04-28T16:45:23 -> 2020-04-28T16:46:23\n",
141+
"Diff times: 2020-04-28T16:46:23 -> 2020-04-28T16:47:23\n",
142+
"Diff times: 2020-04-28T16:47:23 -> 2020-04-28T16:48:23\n",
143+
"Diff times: 2020-04-28T16:48:23 -> 2020-04-28T16:49:23\n",
144+
"Diff times: 2020-04-28T16:49:23 -> 2020-04-28T16:50:23\n",
145+
"Diff times: 2020-04-28T16:50:23 -> 2020-04-28T16:51:23\n",
146+
"Diff times: 2020-04-28T16:51:23 -> 2020-04-28T16:52:23\n",
147+
"Found 261 file changes.\n"
148+
]
149+
}
150+
],
151+
"source": [
152+
"diff_list = diff_snaps(rc, 'EveryMinuteForDiffs')\n",
153+
"print(\"Found %s file changes.\" % len(diff_list))"
154+
]
155+
},
156+
{
157+
"cell_type": "code",
158+
"execution_count": 111,
159+
"metadata": {},
160+
"outputs": [
161+
{
162+
"name": "stdout",
163+
"output_type": "stream",
164+
"text": [
165+
"Ops: CREATE:114 | MODIFY:133 | DELETE:17\n",
166+
"Diff count: 9\n",
167+
"Owner count: 6\n"
168+
]
169+
}
170+
],
171+
"source": [
172+
"owners = {}\n",
173+
"ops = {}\n",
174+
"diffs = {}\n",
175+
"for d in diff_list:\n",
176+
" owners[d['owner']] = 1\n",
177+
" if d['op'] not in ops:\n",
178+
" ops[d['op']] = 1\n",
179+
" ops[d['op']] += 1\n",
180+
" diffs[d['snapshot_id']] = 1\n",
181+
"print(\"Ops: %s\" % ' | '.join([\"%s:%s\" % (k,v) for k, v in ops.items()]))\n",
182+
"print(\"Diff count: %s\" % len(diffs))\n",
183+
"print(\"Owner count: %s\" % len(owners))"
184+
]
185+
},
186+
{
187+
"cell_type": "code",
188+
"execution_count": null,
189+
"metadata": {},
190+
"outputs": [],
191+
"source": []
192+
}
193+
],
194+
"metadata": {
195+
"kernelspec": {
196+
"display_name": "Python 2",
197+
"language": "python",
198+
"name": "python2"
199+
},
200+
"language_info": {
201+
"codemirror_mode": {
202+
"name": "ipython",
203+
"version": 2
204+
},
205+
"file_extension": ".py",
206+
"mimetype": "text/x-python",
207+
"name": "python",
208+
"nbconvert_exporter": "python",
209+
"pygments_lexer": "ipython2",
210+
"version": "2.7.16"
211+
}
212+
},
213+
"nbformat": 4,
214+
"nbformat_minor": 4
215+
}

‎notebooks/System Metrics Summary.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"metadata": {},
66
"source": [
77
"### Pulling system data and metrics from Qumulo\n",
8-
"#### *This is an unsupported approach for pulling metrics data from a Qumulo cluster*"
8+
"#### *This is an unsupported and highly experimental approach for pulling metrics data from a Qumulo cluster*"
99
]
1010
},
1111
{

0 commit comments

Comments
 (0)