Changed types for some DB columns to increase performance of indexes

README.md

@@ -20,6 +20,7 @@ Returns data formatted for the 'daily' parallel coordinates view.
 
 URL parameters:
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are. 
 * 'day' [Mandatory]: in the format YYYY-MM-DD.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
 * 'type' [Optional; default: 'celsius']: type of sensor.
@@ -28,28 +29,35 @@ URL parameters:
 
 Returns data formatted for the 'weekly' parallel coordinates view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'week' [Optional; default: 1]: week number, 1-52.
 * 'year' [Optional; default: 2023]: year.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
 * 'type' [Optional; default: 'celsius']: type of sensor.
+* 'hour' [Optional; default: '0']: the hour returned for each day. 
 
 ### /parallel/monthly
 
 Returns data formatted for the 'monthly' parallel coordinates view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'month' [Optional; default: 1]: month number, 1-12.
 * 'year' [Optional; default: 2023]: year.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
 * 'type' [Optional; default: 'celsius']: type of sensor.
+* 'weekday' [Optional; default: 1]: which day of the week (0 for Mon -> 6 for Sun) is returned.
+* 'hour' [Optional; default: '0']: the hour returned for each day. 
 
 ### /parallel/yearly
 
 Returns data formatted for the 'yearly' parallel coordinates view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'year' [Optional; default: 2023]: year.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
 * 'type' [Optional; default: 'celsius']: type of sensor.
-
+* 'day' [Optional; default: 1]: which day of the month (1-28)
+* 'hour' [Optional; default: '0']: the hour returned for each day. 
 
 ## Grid
 
@@ -57,27 +65,33 @@ Returns data formatted for the 'yearly' parallel coordinates view.
 
 Returns data formatted for the 'weekly' grid view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'week' [Optional; default: 1]: week number, 1-52.
 * 'year' [Optional; default: 2023]: year.
 * 'sensor' [Optional; default: 'S1:1']: name of the sensor.
 * 'type' [Optional; default: 'celsius']: type of sensor.
+* 'hour' [Optional; default: '0']: the hour returned for each day. 
 
 ### /grid/monthly
 
 Returns data formatted for the 'monthly' grid view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'month' [Optional; default: 1]: month number, 1-12.
 * 'year' [Optional; default: 2023]: year.
 * 'sensor' [Optional; default: 'S1:1']: name of the sensor.
 * 'type' [Optional; default: 'celsius']: type of sensor.
+* 'hour' [Optional; default: '0']: the hour returned for each day.
 
 ### /grid/yearly
 
 Returns data formatted for the 'yearly' grid view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'year' [Optional; default: 2023]: year.
 * 'sensor' [Optional; default: 'S1:1']: name of the sensor.
 * 'type' [Optional; default: 'celsius']: type of sensor.
+* 'hour' [Optional; default: '0']: the hour returned for each day.
 
 
 ## Horizon
@@ -86,6 +100,7 @@ Returns data formatted for the 'yearly' grid view.
 
 Returns data formatted for the 'daily' horizon view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'day' [Optional; default: '2023-01-01']: in the format YYYY-MM-DD.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
 * 'type' [Optional; default: 'celsius']: type of sensor.
@@ -94,6 +109,7 @@ Returns data formatted for the 'daily' horizon view.
 
 Returns data formatted for the 'weekly' horizon view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'week' [Optional; default: 1]: week number, 1-52.
 * 'year' [Optional; default: 2023]: year.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
@@ -103,6 +119,7 @@ Returns data formatted for the 'weekly' horizon view.
 
 Returns data formatted for the 'monthly' horizon view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'month' [Optional; default: 1]: month number, 1-12.
 * 'year' [Optional; default: 2023]: year.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
@@ -112,12 +129,13 @@ Returns data formatted for the 'monthly' horizon view.
 
 Returns data formatted for the 'yearly' horizon view.
 
+* 'hus' [Mandatory; default: 'Charlie']: the house where the sensors are.
 * 'year' [Optional; default: 2023]: year.
 * 'family' [Optional; default: 'S1']: name of family of sensors.
 * 'type' [Optional; default: 'celsius']: type of sensor.
 
 
-*NOTE*: For 2023 the data goes only up to August (8 months).
+*NOTE*: For 2023 the data currently goes only up to August (8 months).
 
 
 ## Saved Flexiboards

py/import.py

@@ -71,10 +71,16 @@ for hus_name in next(os.walk(BASEDIR))[1]:
 
 				# Create the table
 				col_names = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'Second']
+				col_types = ['INTEGER'] * 6
 				col_names.extend(f'{family_names[i]}.{sensor_names[i]}.{units[i]}' for i in range(3, len(family_names)))
 				#print(col_names)
+				col_types.extend(['TEXT'] * (len(family_names) - 3))
 
-				cur.execute(f"CREATE TABLE {table_name}{*col_names,}")
+				typed_cols = ', '.join(f'"{col_names[i]}" {col_types[i]}' for i in range(len(col_names)))
+
+				create_query = f"CREATE TABLE {table_name}({typed_cols})"
+
+				cur.execute(f"CREATE TABLE {table_name}({typed_cols})")
 				con.commit()
 
 				cur.execute(f'CREATE INDEX idx_{card}_Year ON {table_name} (Year)')
@@ -97,9 +103,9 @@ for hus_name in next(os.walk(BASEDIR))[1]:
 					header2 = next(reader)
 				else:
 					# First split the date into three columns (y,m,d)
-					row = line[0].split('-')
+					row = [int(x) for x in line[0].split('-')]
 					# Then split the time into three columns (h,m,s)
-					row.extend(line[1].split(':'))
+					row.extend(int(x) for x in line[1].split(':'))
 					# Then add the data (without the date, time, and empty columns)
 					row.extend(tuple(compress(line, not_empty))[2:])
 

py/main.py

@@ -109,7 +109,7 @@ def parallel_daily():
     myxs = {}
 
     for data_point in res_all:
-        point_id = ':'.join(data_point[3:6])
+        point_id = ':'.join(format(y, '02d') for y in data_point[3:6])
         myxs[point_id] = 'pos1'
         data_point = [point_id, *data_point[6:]]
         sample.append(data_point)
@@ -179,9 +179,12 @@ def parallel_monthly():
     typ = request.args.get('type', 'celsius')
     mon = int(request.args.get('month', '1'))
     yea = int(request.args.get('year', '2023'))
-    wda = int(request.args.get('weekday', 1))
+    wda = int(request.args.get('weekday', 0))
     hou = int(request.args.get('hour', '0'))
 
+    if wda < 0 or wda > 6:
+        wda = 0
+
     # Make sure format is right
     hou = f'{hou:02d}'
 
@@ -189,7 +192,10 @@ def parallel_monthly():
     #    return 'ERROR: You need to at least specify the parameters "week" and "year".'
 
     # first days of the 4 weeks
-    first = date(yea, mon, wda)
+    first = date(yea, mon, 1)
+    while first.weekday() != wda:
+        first = first + timedelta(days=1)
+
     days = [str(first + timedelta(days=7*x)) for x in range(4)]
 
     # Get the right sensors for the family
@@ -215,7 +221,7 @@ def parallel_monthly():
     myxs = {}
     # group by day; take first of each group    
     for data_point in res_all:        
-        point_id = '-'.join(data_point[:3])
+        point_id = '-'.join(format(y, '02d') for y in data_point[:3])
         myxs[point_id] = 'pos1'
         data_point = [point_id, *data_point[6:]]
         sample.append(data_point)
@@ -290,8 +296,8 @@ def grid_yearly():
     res_all = cur.fetchall()
 
     out = []
-    out.append(['-'.join(x[:3]) for x in res_all])
-    out.append([':'.join(x[3:6]) for x in res_all])
+    out.append(['-'.join(format(y, '02d') for y in x[:3]) for x in res_all])
+    out.append([':'.join(format(y, '02d') for y in x[3:6]) for x in res_all])
     out.append([x[6] for x in res_all])
     
     return out
@@ -325,8 +331,8 @@ def grid_monthly():
     res_all = cur.fetchall()
 
     out = []
-    out.append(['-'.join(x[:3]) for x in res_all])
-    out.append([':'.join(x[3:6]) for x in res_all])
+    out.append(['-'.join(format(y, '02d') for y in x[:3]) for x in res_all])
+    out.append([':'.join(format(y, '02d') for y in x[3:6]) for x in res_all])
     out.append([x[6] for x in res_all])
 
     return out
@@ -370,8 +376,8 @@ def grid_weekly():
     res_all = cur.fetchall()
 
     out = []
-    out.append(['-'.join(x[:3]) for x in res_all])
-    out.append([':'.join(x[3:6]) for x in res_all])
+    out.append(['-'.join(format(y, '02d') for y in x[:3]) for x in res_all])
+    out.append([':'.join(format(y, '02d') for y in x[3:6]) for x in res_all])
     out.append([x[6] for x in res_all])
 
     return out
@@ -417,8 +423,8 @@ def horizon_yearly():
     for i, sensor_name in enumerate(out['sensor_names']):
         sensor_data_sorted = sorted(aux_data[sensor_name])        
         if i == 0:
-            out['days'] = ['-'.join(x[:3]) for x in sensor_data_sorted]
-            out['times'] = [':'.join(x[3:6]) for x in sensor_data_sorted]
+            out['days'] = ['-'.join(format(y, '02d') for y in x[:3]) for x in sensor_data_sorted]
+            out['times'] = [':'.join(format(y, '02d') for y in x[3:6]) for x in sensor_data_sorted]
         out[sensor_name] = [x[6] for x in sensor_data_sorted]
 
     return out
@@ -459,8 +465,8 @@ def horizon_monthly():
     # Important: this only works assuming that res_all is correctly sorted
 
     for row in res_all:
-        out['days'].append('-'.join(row[:3]))
-        out['times'].append(':'.join(row[3:6]))
+        out['days'].append('-'.join(format(y, '02d') for y in row[:3]))
+        out['times'].append(':'.join(format(y, '02d') for y in row[3:6]))
         for i in range(6, len(row)):
             out[out['sensor_names'][i-6]].append(row[i])
 
@@ -505,8 +511,8 @@ def horizon_weekly():
     # Important: this only works assuming that res_all is correctly sorted
 
     for row in res_all:
-        out['days'].append('-'.join(row[:3]))
-        out['times'].append(':'.join(row[3:6]))
+        out['days'].append('-'.join(format(y, '02d') for y in row[:3]))
+        out['times'].append(':'.join(format(y, '02d') for y in row[3:6]))
         for i in range(6, len(row)):
             out[out['sensor_names'][i-6]].append(row[i])
 
@@ -546,8 +552,8 @@ def horizon_daily():
     # Important: this only works assuming that res_all is correctly sorted
 
     for row in res_all:
-        out['days'].append('-'.join(row[:3]))
-        out['times'].append(':'.join(row[3:6]))
+        out['days'].append('-'.join(format(y, '02d') for y in row[:3]))
+        out['times'].append(':'.join(format(y, '02d') for y in row[3:6]))
         for i in range(6, len(row)):
             out[out['sensor_names'][i-6]].append(row[i])