{"sha":"749810df23875e9eb9c8bac499e6c5ac630ffe96","node_id":"MDY6Q29tbWl0NDgzMDU3NTQ6NzQ5ODEwZGYyMzg3NWU5ZWI5YzhiYWM0OTllNmM1YWM2MzBmZmU5Ng==","commit":{"author":{"name":"Chris J. Karr","email":"chris@audacious-software.com","date":"2018-12-14T00:36:44Z"},"committer":{"name":"Chris J. Karr","email":"chris@audacious-software.com","date":"2018-12-14T00:36:44Z"},"message":"Added date ranges to data export.\n\n* Updated misc. exports to use system timezone when reporting dates.","tree":{"sha":"004a989840503523767d3a3a76e410addea30cc0","url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/git/trees/004a989840503523767d3a3a76e410addea30cc0"},"url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/git/commits/749810df23875e9eb9c8bac499e6c5ac630ffe96","comment_count":0,"verification":{"verified":false,"reason":"unsigned","signature":null,"payload":null}},"url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/commits/749810df23875e9eb9c8bac499e6c5ac630ffe96","html_url":"https://github.com/audacious-software/PassiveDataKit-Django/commit/749810df23875e9eb9c8bac499e6c5ac630ffe96","comments_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/commits/749810df23875e9eb9c8bac499e6c5ac630ffe96/comments","author":{"login":"audaciouscode","id":1141048,"node_id":"MDQ6VXNlcjExNDEwNDg=","avatar_url":"https://avatars.githubusercontent.com/u/1141048?v=4","gravatar_id":"","url":"https://api.github.com/users/audaciouscode","html_url":"https://github.com/audaciouscode","followers_url":"https://api.github.com/users/audaciouscode/followers","following_url":"https://api.github.com/users/audaciouscode/following{/other_user}","gists_url":"https://api.github.com/users/audaciouscode/gists{/gist_id}","starred_url":"https://api.github.com/users/audaciouscode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/audaciouscode/subscriptions","organizations_url":"https://api.github.com/users/audaciouscode/orgs","repos_url":"https://api.github.com/users/audaciouscode/repos","events_url":"https://api.github.com/users/audaciouscode/events{/privacy}","received_events_url":"https://api.github.com/users/audaciouscode/received_events","type":"User","site_admin":false},"committer":{"login":"audaciouscode","id":1141048,"node_id":"MDQ6VXNlcjExNDEwNDg=","avatar_url":"https://avatars.githubusercontent.com/u/1141048?v=4","gravatar_id":"","url":"https://api.github.com/users/audaciouscode","html_url":"https://github.com/audaciouscode","followers_url":"https://api.github.com/users/audaciouscode/followers","following_url":"https://api.github.com/users/audaciouscode/following{/other_user}","gists_url":"https://api.github.com/users/audaciouscode/gists{/gist_id}","starred_url":"https://api.github.com/users/audaciouscode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/audaciouscode/subscriptions","organizations_url":"https://api.github.com/users/audaciouscode/orgs","repos_url":"https://api.github.com/users/audaciouscode/repos","events_url":"https://api.github.com/users/audaciouscode/events{/privacy}","received_events_url":"https://api.github.com/users/audaciouscode/received_events","type":"User","site_admin":false},"parents":[{"sha":"c54efb8a7560b99f2d2231fd99ab023d5ba9bbb5","url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/commits/c54efb8a7560b99f2d2231fd99ab023d5ba9bbb5","html_url":"https://github.com/audacious-software/PassiveDataKit-Django/commit/c54efb8a7560b99f2d2231fd99ab023d5ba9bbb5"}],"stats":{"total":762,"additions":612,"deletions":150},"files":[{"sha":"045eb860dcc73655155b42899c5a78b5da6e610f","filename":"generators/pdk_app_event.py","status":"modified","additions":15,"deletions":3,"changes":18,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_app_event.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_app_event.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/generators%2Fpdk_app_event.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -7,6 +7,8 @@\n import tempfile\n import time\n \n+import pytz\n+\n from django.conf import settings\n from django.template.loader import render_to_string\n from django.utils import timezone\n@@ -22,7 +24,7 @@ def extract_secondary_identifier(properties):\n \n return None\n \n-def compile_report(generator, sources):\n+def compile_report(generator, sources, data_start=None, data_end=None): # pylint: disable=too-many-locals\n filename = tempfile.gettempdir() + '/pdk_' + generator + '.txt'\n \n with open(filename, 'w') as outfile:\n@@ -37,7 +39,15 @@ def compile_report(generator, sources):\n ])\n \n for source in sources:\n- points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created') # pylint: disable=no-member,line-too-long\n+ points = DataPoint.objects.filter(source=source, generator_identifier=generator)\n+\n+ if data_start is not None:\n+ points = points.filter(created__gte=data_start)\n+\n+ if data_end is not None:\n+ points = points.filter(created__lte=data_end)\n+\n+ points = points.order_by('source', 'created')\n \n index = 0\n count = points.count()\n@@ -46,9 +56,11 @@ def compile_report(generator, sources):\n for point in points[index:(index + 5000)]:\n row = []\n \n+ created = point.created.astimezone(pytz.timezone(settings.TIME_ZONE))\n+\n row.append(point.source)\n row.append(calendar.timegm(point.created.utctimetuple()))\n- row.append(point.created.isoformat())\n+ row.append(created.isoformat())\n \n properties = {}\n "},{"sha":"e2805963be78bf93760bbeeb5ccfd8a032899e33","filename":"generators/pdk_device_battery.py","status":"modified","additions":75,"deletions":53,"changes":128,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_device_battery.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_device_battery.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/generators%2Fpdk_device_battery.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -1,12 +1,21 @@\n # pylint: disable=line-too-long, no-member\n \n+import csv\n import calendar\n import datetime\n import json\n+import os\n+import tempfile\n+\n+from zipfile import ZipFile\n+\n+import arrow\n+import pytz\n \n from django.conf import settings\n from django.template.loader import render_to_string\n from django.utils import timezone\n+from django.utils.text import slugify\n \n from ..models import DataPoint\n \n@@ -139,56 +148,69 @@ def data_table(source, generator):\n return render_to_string('generators/pdk_device_battery_table_template.html', context)\n \n \n-# def compile_report(generator, sources): # pylint: disable=too-many-locals\n-# timestamp = arrow.get()\n-# filename = tempfile.gettempdir() + '/pdk_export_' + str(timestamp.seconds) + '.' + \\\n-# str(timestamp.seconds / 1e6) + '.zip'\n-#\n-# with ZipFile(filename, 'w') as export_file:\n-# for secondary_identifier in SECONDARY_FIELDS:\n-# secondary_filename = tempfile.gettempdir() + '/' + generator + '-' + \\\n-# secondary_identifier + '.txt'\n-#\n-# with open(secondary_filename, 'w') as outfile:\n-# writer = csv.writer(outfile, delimiter='\\t')\n-#\n-# columns = [\n-# 'Source',\n-# 'Created Timestamp',\n-# 'Created Date',\n-# ]\n-#\n-# for column in SECONDARY_FIELDS[secondary_identifier]:\n-# columns.append(column)\n-#\n-# writer.writerow(columns)\n-#\n-# for source in sources:\n-# points = DataPoint.objects.filter(source=source, generator_identifier=generator, secondary_identifier=secondary_identifier).order_by('source', 'created') # pylint: disable=no-member,line-too-long\n-#\n-# index = 0\n-# count = points.count()\n-#\n-# while index < count:\n-# for point in points[index:(index + 5000)]:\n-# row = []\n-#\n-# row.append(point.source)\n-# row.append(calendar.timegm(point.created.utctimetuple()))\n-# row.append(point.created.isoformat())\n-#\n-# properties = point.fetch_properties()\n-#\n-# for column in SECONDARY_FIELDS[secondary_identifier]:\n-# if column in properties:\n-# row.append(properties[column])\n-# else:\n-# row.append('')\n-#\n-# writer.writerow(row)\n-#\n-# index += 5000\n-#\n-# export_file.write(secondary_filename, secondary_filename.split('/')[-1])\n-#\n-# return filename\n+def compile_report(generator, sources, data_start=None, data_end=None): # pylint: disable=too-many-locals\n+ now = arrow.get()\n+ filename = tempfile.gettempdir() + '/pdk_export_' + str(now.timestamp) + str(now.microsecond / 1e6) + '.zip'\n+\n+ with ZipFile(filename, 'w') as export_file:\n+ for source in sources:\n+ identifier = slugify(generator + '__' + source)\n+\n+ secondary_filename = tempfile.gettempdir() + '/' + identifier + '.txt'\n+\n+ with open(secondary_filename, 'w') as outfile:\n+ writer = csv.writer(outfile, delimiter='\\t')\n+\n+ columns = [\n+ 'Source',\n+ 'Created Timestamp',\n+ 'Created Date',\n+ 'Level',\n+ 'Scale',\n+ 'Plugged',\n+ 'Health',\n+ 'Voltage',\n+ 'Technology',\n+ 'Present',\n+ 'Temperature',\n+ ]\n+\n+ writer.writerow(columns)\n+\n+ points = DataPoint.objects.filter(source=source, generator_identifier=generator)\n+\n+ if data_start is not None:\n+ points = points.filter(created__gte=data_start)\n+\n+ if data_end is not None:\n+ points = points.filter(created__lte=data_end)\n+\n+ points = points.order_by('source', 'created')\n+\n+ for point in points:\n+ properties = point.fetch_properties()\n+\n+ row = []\n+\n+ created = point.created.astimezone(pytz.timezone(settings.TIME_ZONE))\n+\n+ row.append(point.source)\n+ row.append(calendar.timegm(point.created.utctimetuple()))\n+ row.append(created.isoformat())\n+\n+ row.append(properties['level'])\n+ row.append(properties['scale'])\n+ row.append(properties['plugged'])\n+ row.append(properties['health'])\n+ row.append(properties['voltage'])\n+ row.append(properties['technology'])\n+ row.append(properties['present'])\n+ row.append(properties['temperature'])\n+\n+ writer.writerow(row)\n+\n+ export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt')\n+\n+ os.remove(secondary_filename)\n+\n+ return filename"},{"sha":"f630efb65c2702ddb485210daa5028c79cd0a0c4","filename":"generators/pdk_foreground_application.py","status":"modified","additions":68,"deletions":53,"changes":121,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_foreground_application.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_foreground_application.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/generators%2Fpdk_foreground_application.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -1,12 +1,22 @@\n # pylint: disable=line-too-long, no-member\n \n+import calendar\n+import csv\n import datetime\n import json\n+import os\n+import tempfile\n import time\n \n+from zipfile import ZipFile\n+\n+import arrow\n+import pytz\n+\n from django.conf import settings\n from django.template.loader import render_to_string\n from django.utils import timezone\n+from django.utils.text import slugify\n \n from ..models import DataPoint\n \n@@ -142,56 +152,61 @@ def data_table(source, generator): # pylint: disable=too-many-locals\n \n return render_to_string('pdk_foreground_application_table_template.html', context)\n \n-# def compile_report(generator, sources): # pylint: disable=too-many-locals\n-# timestamp = arrow.get()\n-# filename = tempfile.gettempdir() + '/pdk_export_' + str(timestamp.seconds) + '.' + \\\n-# str(timestamp.seconds / 1e6) + '.zip'\n-#\n-# with ZipFile(filename, 'w') as export_file:\n-# for secondary_identifier in SECONDARY_FIELDS:\n-# secondary_filename = tempfile.gettempdir() + '/' + generator + '-' + \\\n-# secondary_identifier + '.txt'\n-#\n-# with open(secondary_filename, 'w') as outfile:\n-# writer = csv.writer(outfile, delimiter='\\t')\n-#\n-# columns = [\n-# 'Source',\n-# 'Created Timestamp',\n-# 'Created Date',\n-# ]\n-#\n-# for column in SECONDARY_FIELDS[secondary_identifier]:\n-# columns.append(column)\n-#\n-# writer.writerow(columns)\n-#\n-# for source in sources:\n-# points = DataPoint.objects.filter(source=source, generator_identifier=generator, secondary_identifier=secondary_identifier).order_by('source', 'created') # pylint: disable=no-member,line-too-long\n-#\n-# index = 0\n-# count = points.count()\n-#\n-# while index < count:\n-# for point in points[index:(index + 5000)]:\n-# row = []\n-#\n-# row.append(point.source)\n-# row.append(calendar.timegm(point.created.utctimetuple()))\n-# row.append(point.created.isoformat())\n-#\n-# properties = point.fetch_properties()\n-#\n-# for column in SECONDARY_FIELDS[secondary_identifier]:\n-# if column in properties:\n-# row.append(properties[column])\n-# else:\n-# row.append('')\n-#\n-# writer.writerow(row)\n-#\n-# index += 5000\n-#\n-# export_file.write(secondary_filename, secondary_filename.split('/')[-1])\n-#\n-# return filename\n+def compile_report(generator, sources, data_start=None, data_end=None): # pylint: disable=too-many-locals\n+ now = arrow.get()\n+ filename = tempfile.gettempdir() + '/pdk_export_' + str(now.timestamp) + str(now.microsecond / 1e6) + '.zip'\n+\n+ with ZipFile(filename, 'w') as export_file:\n+ for source in sources:\n+ identifier = slugify(generator + '__' + source)\n+\n+ secondary_filename = tempfile.gettempdir() + '/' + identifier + '.txt'\n+\n+ with open(secondary_filename, 'w') as outfile:\n+ writer = csv.writer(outfile, delimiter='\\t')\n+\n+ columns = [\n+ 'Source',\n+ 'Created Timestamp',\n+ 'Created Date',\n+ 'Application',\n+ 'Duration',\n+ 'Screen Active',\n+ ]\n+\n+ writer.writerow(columns)\n+\n+ points = DataPoint.objects.filter(source=source, generator_identifier=generator)\n+\n+ if data_start is not None:\n+ points = points.filter(created__gte=data_start)\n+\n+ if data_end is not None:\n+ points = points.filter(created__lte=data_end)\n+\n+ points = points.order_by('source', 'created')\n+\n+ for point in points:\n+ properties = point.fetch_properties()\n+\n+ row = []\n+\n+ created = point.created.astimezone(pytz.timezone(settings.TIME_ZONE))\n+\n+ row.append(point.source)\n+ row.append(calendar.timegm(point.created.utctimetuple()))\n+ row.append(created.isoformat())\n+\n+ properties = point.fetch_properties()\n+\n+ row.append(properties['application'])\n+ row.append(str(properties['duration']))\n+ row.append(str(properties['screen_active']))\n+\n+ writer.writerow(row)\n+\n+ export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt')\n+\n+ os.remove(secondary_filename)\n+\n+ return filename"},{"sha":"90084b4ccd083969bc6d22e5b75684ce30d2a5c5","filename":"generators/pdk_screen_state.py","status":"modified","additions":13,"deletions":8,"changes":21,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_screen_state.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_screen_state.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/generators%2Fpdk_screen_state.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -74,7 +74,7 @@ def data_table(source, generator):\n \n return render_to_string('pdk_screen_state_table_template.html', context)\n \n-def compile_report(generator, sources): # pylint: disable=too-many-locals\n+def compile_report(generator, sources, data_start=None, data_end=None): # pylint: disable=too-many-locals\n now = arrow.get()\n filename = tempfile.gettempdir() + '/pdk_export_' + str(now.timestamp) + str(now.microsecond / 1e6) + '.zip'\n \n@@ -91,26 +91,31 @@ def compile_report(generator, sources): # pylint: disable=too-many-locals\n 'Source',\n 'Created Timestamp',\n 'Created Date',\n- 'Recorded Timestamp',\n- 'Recorded Date',\n 'Screen State'\n ]\n \n writer.writerow(columns)\n \n- points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created')\n+ points = DataPoint.objects.filter(source=source, generator_identifier=generator)\n+\n+ if data_start is not None:\n+ points = points.filter(created__gte=data_start)\n+\n+ if data_end is not None:\n+ points = points.filter(created__lte=data_end)\n+\n+ points = points.order_by('source', 'created')\n \n for point in points:\n properties = point.fetch_properties()\n \n row = []\n \n+ created = point.created.astimezone(pytz.timezone(settings.TIME_ZONE))\n+\n row.append(point.source)\n row.append(calendar.timegm(point.created.utctimetuple()))\n- row.append(point.created.isoformat())\n-\n- row.append(calendar.timegm(point.recorded.utctimetuple()))\n- row.append(point.recorded.isoformat())\n+ row.append(created.isoformat())\n \n row.append(properties['state'])\n "},{"sha":"7ec65dfcba334252519a6f8f20621645bba235ed","filename":"generators/pdk_system_status.py","status":"modified","additions":20,"deletions":8,"changes":28,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_system_status.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_system_status.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/generators%2Fpdk_system_status.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -11,6 +11,7 @@\n from zipfile import ZipFile\n \n import arrow\n+import pytz\n \n from django.conf import settings\n from django.template.loader import render_to_string\n@@ -54,7 +55,7 @@ def data_table(source, generator):\n \n return render_to_string('generators/pdk_device_system_status_table_template.html', context)\n \n-def compile_report(generator, sources): # pylint: disable=too-many-locals\n+def compile_report(generator, sources, data_start=None, data_end=None): # pylint: disable=too-many-locals\n now = arrow.get()\n filename = tempfile.gettempdir() + '/pdk_export_' + str(now.timestamp) + str(now.microsecond / 1e6) + '.zip'\n \n@@ -71,30 +72,36 @@ def compile_report(generator, sources): # pylint: disable=too-many-locals\n 'Source',\n 'Created Timestamp',\n 'Created Date',\n- 'Recorded Timestamp',\n- 'Recorded Date',\n 'Available Storage',\n 'Other Storage',\n 'App Storage',\n 'Total Storage',\n 'App Runtime',\n+ 'System Runtime',\n ]\n \n writer.writerow(columns)\n \n- points = DataPoint.objects.filter(source=source, generator_identifier=generator).order_by('created')\n+ points = DataPoint.objects.filter(source=source, generator_identifier=generator)\n+\n+ if data_start is not None:\n+ points = points.filter(created__gte=data_start)\n+\n+ if data_end is not None:\n+ points = points.filter(created__lte=data_end)\n+\n+ points = points.order_by('source', 'created')\n \n for point in points:\n properties = point.fetch_properties()\n \n row = []\n \n+ created = point.created.astimezone(pytz.timezone(settings.TIME_ZONE))\n+\n row.append(point.source)\n row.append(calendar.timegm(point.created.utctimetuple()))\n- row.append(point.created.isoformat())\n-\n- row.append(calendar.timegm(point.recorded.utctimetuple()))\n- row.append(point.recorded.isoformat())\n+ row.append(created.isoformat())\n \n row.append(properties['storage_available'])\n row.append(properties['storage_other'])\n@@ -106,6 +113,11 @@ def compile_report(generator, sources): # pylint: disable=too-many-locals\n else:\n row.append(None)\n \n+ if 'system_runtime' in properties:\n+ row.append(properties['system_runtime'])\n+ else:\n+ row.append(None)\n+\n writer.writerow(row)\n \n export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt')"},{"sha":"880c38aa25a9480f71b94743e68547698f513c59","filename":"generators/pdk_user.py","status":"added","additions":131,"deletions":0,"changes":131,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_user.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/generators%2Fpdk_user.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/generators%2Fpdk_user.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -0,0 +1,131 @@\n+# pylint: disable=line-too-long, no-member\n+\n+import calendar\n+import csv\n+import datetime\n+import json\n+import os\n+import tempfile\n+\n+from zipfile import ZipFile\n+\n+import arrow\n+import pytz\n+\n+from django.conf import settings\n+from django.template.loader import render_to_string\n+from django.utils import timezone\n+from django.utils.text import slugify\n+\n+from ..models import DataPoint\n+\n+def generator_name(identifier): # pylint: disable=unused-argument\n+ return 'User State'\n+\n+def visualization(source, generator):\n+ context = {}\n+ context['source'] = source\n+ context['generator_identifier'] = generator\n+\n+ now = timezone.now()\n+\n+ days = []\n+\n+ start = datetime.datetime(now.year, now.month, now.day, 0, 0, 0, 0, pytz.timezone(settings.TIME_ZONE))\n+\n+ while len(days) < 7:\n+ end = start + datetime.timedelta(days=1)\n+\n+ points = DataPoint.objects.filter(source=source.identifier, generator_identifier=generator, created__gt=start, created__lte=end).order_by('created')\n+\n+ point_list = []\n+\n+ for point in points:\n+ properties = point.fetch_properties()\n+\n+ data_obj = {\n+ 'timestamp': calendar.timegm(point.created.utctimetuple()),\n+ 'value': properties['mode']\n+ }\n+\n+ point_list.append(data_obj)\n+\n+ day = {\n+ 'points': point_list,\n+ 'start_txt': start.isoformat(),\n+ 'start': calendar.timegm(start.utctimetuple()),\n+ 'end': calendar.timegm(end.utctimetuple())\n+ }\n+\n+ days.insert(0, day)\n+\n+ start = start - datetime.timedelta(days=1)\n+\n+ context['days'] = json.dumps(days)\n+\n+ return render_to_string('pdk_device_user_template.html', context)\n+\n+def data_table(source, generator):\n+ context = {}\n+ context['source'] = source\n+ context['generator_identifier'] = generator\n+\n+ context['values'] = DataPoint.objects.filter(source=source.identifier, generator_identifier=generator).order_by('-created')[:1000]\n+\n+ return render_to_string('pdk_user_table_template.html', context)\n+\n+def compile_report(generator, sources, data_start=None, data_end=None): # pylint: disable=too-many-locals\n+ now = arrow.get()\n+ filename = tempfile.gettempdir() + '/pdk_export_' + str(now.timestamp) + str(now.microsecond / 1e6) + '.zip'\n+\n+ with ZipFile(filename, 'w') as export_file:\n+ for source in sources:\n+ identifier = slugify(generator + '__' + source)\n+\n+ secondary_filename = tempfile.gettempdir() + '/' + identifier + '.txt'\n+\n+ with open(secondary_filename, 'w') as outfile:\n+ writer = csv.writer(outfile, delimiter='\\t')\n+\n+ columns = [\n+ 'Source',\n+ 'Created Timestamp',\n+ 'Created Date',\n+ 'Mode'\n+ ]\n+\n+ writer.writerow(columns)\n+\n+ points = DataPoint.objects.filter(source=source, generator_identifier=generator)\n+\n+ if data_start is not None:\n+ points = points.filter(created__gte=data_start)\n+\n+ if data_end is not None:\n+ points = points.filter(created__lte=data_end)\n+\n+ points = points.order_by('source', 'created')\n+\n+ for point in points:\n+ properties = point.fetch_properties()\n+\n+ row = []\n+\n+ created = point.created.astimezone(pytz.timezone(settings.TIME_ZONE))\n+\n+ row.append(point.source)\n+ row.append(calendar.timegm(point.created.utctimetuple()))\n+ row.append(created.isoformat())\n+ \n+ if 'mode' in properties:\n+ row.append(properties['mode'])\n+ else:\n+ row.append(None)\n+\n+ writer.writerow(row)\n+\n+ export_file.write(secondary_filename, slugify(generator) + '/' + slugify(source) + '.txt')\n+\n+ os.remove(secondary_filename)\n+\n+ return filename"},{"sha":"d7e41d67355805b92ca4427eb9c712a27167215d","filename":"management/commands/pdk_compile_reports.py","status":"modified","additions":40,"deletions":13,"changes":53,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/management%2Fcommands%2Fpdk_compile_reports.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/management%2Fcommands%2Fpdk_compile_reports.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/management%2Fcommands%2Fpdk_compile_reports.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -10,6 +10,8 @@\n \n import zipstream\n \n+import pytz\n+\n from django.conf import settings\n from django.core.files import File\n from django.core.mail import send_mail\n@@ -25,17 +27,6 @@ class Command(BaseCommand):\n \n def add_arguments(self, parser):\n pass\n-# parser.add_argument('--delete',\n-# action='store_true',\n-# dest='delete',\n-# default=False,\n-# help='Delete data bundles after processing')\n-#\n-# parser.add_argument('--count',\n-# type=int,\n-# dest='bundle_count',\n-# default=100,\n-# help='Number of bundles to process in a single run')\n \n @handle_lock\n def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-branches,too-many-statements\n@@ -59,6 +50,36 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b\n sources = parameters['sources']\n generators = parameters['generators']\n \n+ data_start = None\n+ data_end = None\n+\n+ tz_info = pytz.timezone(settings.TIME_ZONE)\n+\n+ if 'data_start' in parameters and parameters['data_start']:\n+ tokens = parameters['data_start'].split('/')\n+\n+ data_start = datetime.datetime(int(tokens[2]), \\\n+ int(tokens[0]), \\\n+ int(tokens[1]), \\\n+ 0, \\\n+ 0, \\\n+ 0, \\\n+ 0, \\\n+ tz_info)\n+\n+ if 'data_end' in parameters and parameters['data_end']:\n+ tokens = parameters['data_end'].split('/')\n+\n+ data_end = datetime.datetime(int(tokens[2]), \\\n+ int(tokens[0]), \\\n+ int(tokens[1]), \\\n+ 23, \\\n+ 59, \\\n+ 59, \\\n+ 999999, \\\n+ tz_info)\n+\n+\n raw_json = False\n \n if ('raw_data' in parameters) and parameters['raw_data'] is True:\n@@ -97,7 +118,13 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b\n 0, \\\n 0, \\\n first_create.tzinfo) + \\\n- datetime.timedelta(days=1)\n+ datetime.timedelta(days=1)\n+\n+ if data_start is not None and data_start > start:\n+ start = data_start\n+\n+ if data_end is not None and data_end < data_end:\n+ end = data_end\n \n while start <= end:\n day_end = start + datetime.timedelta(days=1)\n@@ -124,7 +151,7 @@ def handle(self, *args, **options): # pylint: disable=too-many-locals,too-many-b\n try:\n pdk_api = importlib.import_module(app + '.pdk_api')\n \n- output_file = pdk_api.compile_report(generator, sources)\n+ output_file = pdk_api.compile_report(generator, sources, data_start=data_start, data_end=data_end)\n \n if output_file is not None:\n if output_file.lower().endswith('.zip'):"},{"sha":"ccfe51d01b376688fc361a7df6f34ec7854d57f9","filename":"migrations/0040_auto_20181213_2144.py","status":"added","additions":31,"deletions":0,"changes":31,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/migrations%2F0040_auto_20181213_2144.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/migrations%2F0040_auto_20181213_2144.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/migrations%2F0040_auto_20181213_2144.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -0,0 +1,31 @@\n+# pylint: skip-file\n+\n+# -*- coding: utf-8 -*-\n+# Generated by Django 1.11.15 on 2018-12-13 21:44\n+from __future__ import unicode_literals\n+\n+from django.db import migrations, models\n+\n+\n+class Migration(migrations.Migration):\n+\n+ dependencies = [\n+ ('passive_data_kit', '0039_dataservermetadatum_last_updated'),\n+ ]\n+\n+ operations = [\n+ migrations.AlterModelOptions(\n+ name='dataservermetadatum',\n+ options={'verbose_name_plural': 'data server metadata'},\n+ ),\n+ migrations.AddField(\n+ model_name='reportjob',\n+ name='data_end',\n+ field=models.DateTimeField(blank=True, null=True),\n+ ),\n+ migrations.AddField(\n+ model_name='reportjob',\n+ name='data_start',\n+ field=models.DateTimeField(blank=True, null=True),\n+ ),\n+ ]"},{"sha":"fe81fe3619e68de5a926acc24c85bcf55976ef89","filename":"migrations/0041_auto_20181213_2231.py","status":"added","additions":25,"deletions":0,"changes":25,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/migrations%2F0041_auto_20181213_2231.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/migrations%2F0041_auto_20181213_2231.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/migrations%2F0041_auto_20181213_2231.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -0,0 +1,25 @@\n+# pylint: skip-file\n+\n+# -*- coding: utf-8 -*-\n+# Generated by Django 1.11.15 on 2018-12-13 22:31\n+from __future__ import unicode_literals\n+\n+from django.db import migrations\n+\n+\n+class Migration(migrations.Migration):\n+\n+ dependencies = [\n+ ('passive_data_kit', '0040_auto_20181213_2144'),\n+ ]\n+\n+ operations = [\n+ migrations.RemoveField(\n+ model_name='reportjob',\n+ name='data_end',\n+ ),\n+ migrations.RemoveField(\n+ model_name='reportjob',\n+ name='data_start',\n+ ),\n+ ]"},{"sha":"4b9a02ec4de654673e9e5163f50ebe28b0efa73e","filename":"models.py","status":"modified","additions":7,"deletions":1,"changes":8,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/models.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/models.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/models.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -597,14 +597,16 @@ class DataPointVisualization(models.Model):\n \n \n class ReportJobManager(models.Manager): # pylint: disable=too-few-public-methods\n- def create_jobs(self, user, sources, generators, export_raw=False): # pylint: disable=too-many-locals, too-many-branches, too-many-statements, no-self-use\n+ def create_jobs(self, user, sources, generators, export_raw=False, data_start=None, data_end=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements, no-self-use, too-many-arguments\n batch_request = ReportJobBatchRequest(requester=user, requested=timezone.now())\n \n params = {}\n \n params['sources'] = sources\n params['generators'] = generators\n params['export_raw'] = export_raw\n+ params['data_start'] = data_start\n+ params['data_end'] = data_end\n \n if install_supports_jsonfield():\n batch_request.parameters = params\n@@ -713,6 +715,8 @@ def process(self): # pylint: disable=too-many-locals, too-many-branches, too-man\n job_params['sources'] = report_sources\n job_params['generators'] = params['generators']\n job_params['raw_data'] = params['export_raw']\n+ job_params['data_start'] = params['data_start']\n+ job_params['data_end'] = params['data_end']\n \n if install_supports_jsonfield():\n job.parameters = job_params\n@@ -734,6 +738,8 @@ def process(self): # pylint: disable=too-many-locals, too-many-branches, too-man\n job_params['sources'] = report_sources\n job_params['generators'] = params['generators']\n job_params['raw_data'] = params['export_raw']\n+ job_params['data_start'] = params['data_start']\n+ job_params['data_end'] = params['data_end']\n \n if install_supports_jsonfield():\n job.parameters = job_params"},{"sha":"a73133dd7840e9d67bf33a3f6afc95ff2f9329ad","filename":"pdk_api.py","status":"modified","additions":2,"deletions":2,"changes":4,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/pdk_api.py","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/pdk_api.py","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/pdk_api.py?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -87,11 +87,11 @@ def data_table(source, generator):\n \n return render_to_string('pdk_generic_viz_template.html', context)\n \n-def compile_report(generator, sources):\n+def compile_report(generator, sources, data_start=None, data_end=None):\n try:\n generator_module = importlib.import_module('.generators.' + generator.replace('-', '_'), package='passive_data_kit')\n \n- output_file = generator_module.compile_report(generator, sources)\n+ output_file = generator_module.compile_report(generator, sources, data_start=data_start, data_end=data_end)\n \n if output_file is not None:\n return output_file"},{"sha":"589339fe3a026bcd810be371c25f9941367de36a","filename":"static/pdk/js/export.js","status":"added","additions":21,"deletions":0,"changes":21,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/static%2Fpdk%2Fjs%2Fexport.js","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/static%2Fpdk%2Fjs%2Fexport.js","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/static%2Fpdk%2Fjs%2Fexport.js?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -0,0 +1,21 @@\n+//Load common code that includes config, then load the app logic for this page.\n+requirejs(['./common'], function (common) {\n+ requirejs([\"bootstrap\", \"bootstrap-datepicker\", \"moment\", \"jquery\"], function (bootstrap, bs_datepicker, moment, jquery)\n+ {\n+ \twindow.$ = jquery;\n+ \twindow.moment = moment;\n+ \t\n+\t\t$.ajaxSetup({ \n+\t\t\tbeforeSend: function(xhr, settings) \n+\t\t\t{\n+\t\t\t\tif (!(/^http:.*/.test(settings.url) || /^https:.*/.test(settings.url))) \n+\t\t\t\t{\n+\t\t\t\t\t// Only send the token to relative URLs i.e. locally.\n+\t\t\t\t\txhr.setRequestHeader(\"X-CSRFToken\", $.cookie('csrftoken'));\n+\t\t\t\t}\n+\t\t\t}\n+\t\t});\n+\n+\t\t$('[data-toggle=\"tooltip\"]').tooltip();\n+\t}); \n+});\n\\ No newline at end of file"},{"sha":"4ba060f6fa3e74f0ff393b6e72739194ea815e3f","filename":"static/pdk/js/lib/bootstrap-datepicker.js","status":"renamed","additions":0,"deletions":0,"changes":0,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/static%2Fpdk%2Fjs%2Flib%2Fbootstrap-datepicker.js","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/static%2Fpdk%2Fjs%2Flib%2Fbootstrap-datepicker.js","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/static%2Fpdk%2Fjs%2Flib%2Fbootstrap-datepicker.js?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","previous_filename":"static/pdk/js/lib/bootstrap-datepicker.min.js"},{"sha":"c8494940681b04cffdf322cfd0183eeb996122e9","filename":"templates/pdk_base.html","status":"modified","additions":1,"deletions":0,"changes":1,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/templates%2Fpdk_base.html","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/templates%2Fpdk_base.html","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/templates%2Fpdk_base.html?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -12,6 +12,7 @@\n \t\t\n \t\t\n \t\t\n+\t\t\n \t\t\n \t\t\n \t\t"},{"sha":"b85ee8461408d5956b71624ab1f87574086dda95","filename":"templates/pdk_device_user_template.html","status":"added","additions":106,"deletions":0,"changes":106,"blob_url":"https://github.com/audacious-software/PassiveDataKit-Django/blob/749810df23875e9eb9c8bac499e6c5ac630ffe96/templates%2Fpdk_device_user_template.html","raw_url":"https://github.com/audacious-software/PassiveDataKit-Django/raw/749810df23875e9eb9c8bac499e6c5ac630ffe96/templates%2Fpdk_device_user_template.html","contents_url":"https://api.github.com/repos/audacious-software/PassiveDataKit-Django/contents/templates%2Fpdk_device_user_template.html?ref=749810df23875e9eb9c8bac499e6c5ac630ffe96","patch":"@@ -0,0 +1,106 @@\n+
\n- Select All Sources\n+ Select All Sources\n
\n \n\n \t \n- \t\n+ \t\n
\n {% endfor %}\n\n- Select All Generators\n+ Select All Generators\n
\n \n\n \t\t\t\t\t\t\t\t \n-\t\t\t\t\t\t\t\t\n+\t\t\t\t\t\t\t\t\n \t\t\t\t\t\t\t
\n \t\t\t\t\t\t{% endfor %}\n \t\t\t\t\t\t\n \t \n- \t\n+ \t\n
\n {% endfor %}\n\n+\t\t\t\t\t Export raw data as JSON\n+\t\t\t\t
\nCreated | \n+Mode | \n+
---|---|
\n+\t\t\t\t\t\t\n+\t\t\t\t\t\t\t{{ row.created.isoformat }}\n+\t\t\t\t\t\t\n+\t\t\t\t\t\t{{ row.created }}\n+\t\t\t\t\t | \n+\t\t\t\t\t{{ props.mode }} | \n+\t\t\t\t