Changed types for some DB columns to increase performance of indexes

py/import.py

@@ -71,10 +71,16 @@ for hus_name in next(os.walk(BASEDIR))[1]:
 
 				# Create the table
 				col_names = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'Second']
+				col_types = ['INTEGER'] * 6
 				col_names.extend(f'{family_names[i]}.{sensor_names[i]}.{units[i]}' for i in range(3, len(family_names)))
 				#print(col_names)
+				col_types.extend(['TEXT'] * (len(family_names) - 3))
 
-				cur.execute(f"CREATE TABLE {table_name}{*col_names,}")
+				typed_cols = ', '.join(f'"{col_names[i]}" {col_types[i]}' for i in range(len(col_names)))
+
+				create_query = f"CREATE TABLE {table_name}({typed_cols})"
+
+				cur.execute(f"CREATE TABLE {table_name}({typed_cols})")
 				con.commit()
 
 				cur.execute(f'CREATE INDEX idx_{card}_Year ON {table_name} (Year)')
@@ -97,9 +103,9 @@ for hus_name in next(os.walk(BASEDIR))[1]:
 					header2 = next(reader)
 				else:
 					# First split the date into three columns (y,m,d)
-					row = line[0].split('-')
+					row = [int(x) for x in line[0].split('-')]
 					# Then split the time into three columns (h,m,s)
-					row.extend(line[1].split(':'))
+					row.extend(int(x) for x in line[1].split(':'))
 					# Then add the data (without the date, time, and empty columns)
 					row.extend(tuple(compress(line, not_empty))[2:])
 

py/main.py

@@ -109,7 +109,7 @@ def parallel_daily():
     myxs = {}
 
     for data_point in res_all:
-        point_id = ':'.join(data_point[3:6])
+        point_id = ':'.join(format(y, '02d') for y in data_point[3:6])
         myxs[point_id] = 'pos1'
         data_point = [point_id, *data_point[6:]]
         sample.append(data_point)
@@ -221,7 +221,7 @@ def parallel_monthly():
     myxs = {}
     # group by day; take first of each group    
     for data_point in res_all:        
-        point_id = '-'.join(data_point[:3])
+        point_id = '-'.join(format(y, '02d') for y in data_point[:3])
         myxs[point_id] = 'pos1'
         data_point = [point_id, *data_point[6:]]
         sample.append(data_point)
@@ -296,8 +296,8 @@ def grid_yearly():
     res_all = cur.fetchall()
 
     out = []
-    out.append(['-'.join(x[:3]) for x in res_all])
-    out.append([':'.join(x[3:6]) for x in res_all])
+    out.append(['-'.join(format(y, '02d') for y in x[:3]) for x in res_all])
+    out.append([':'.join(format(y, '02d') for y in x[3:6]) for x in res_all])
     out.append([x[6] for x in res_all])
     
     return out
@@ -331,8 +331,8 @@ def grid_monthly():
     res_all = cur.fetchall()
 
     out = []
-    out.append(['-'.join(x[:3]) for x in res_all])
-    out.append([':'.join(x[3:6]) for x in res_all])
+    out.append(['-'.join(format(y, '02d') for y in x[:3]) for x in res_all])
+    out.append([':'.join(format(y, '02d') for y in x[3:6]) for x in res_all])
     out.append([x[6] for x in res_all])
 
     return out
@@ -376,8 +376,8 @@ def grid_weekly():
     res_all = cur.fetchall()
 
     out = []
-    out.append(['-'.join(x[:3]) for x in res_all])
-    out.append([':'.join(x[3:6]) for x in res_all])
+    out.append(['-'.join(format(y, '02d') for y in x[:3]) for x in res_all])
+    out.append([':'.join(format(y, '02d') for y in x[3:6]) for x in res_all])
     out.append([x[6] for x in res_all])
 
     return out
@@ -423,8 +423,8 @@ def horizon_yearly():
     for i, sensor_name in enumerate(out['sensor_names']):
         sensor_data_sorted = sorted(aux_data[sensor_name])        
         if i == 0:
-            out['days'] = ['-'.join(x[:3]) for x in sensor_data_sorted]
-            out['times'] = [':'.join(x[3:6]) for x in sensor_data_sorted]
+            out['days'] = ['-'.join(format(y, '02d') for y in x[:3]) for x in sensor_data_sorted]
+            out['times'] = [':'.join(format(y, '02d') for y in x[3:6]) for x in sensor_data_sorted]
         out[sensor_name] = [x[6] for x in sensor_data_sorted]
 
     return out
@@ -465,8 +465,8 @@ def horizon_monthly():
     # Important: this only works assuming that res_all is correctly sorted
 
     for row in res_all:
-        out['days'].append('-'.join(row[:3]))
-        out['times'].append(':'.join(row[3:6]))
+        out['days'].append('-'.join(format(y, '02d') for y in row[:3]))
+        out['times'].append(':'.join(format(y, '02d') for y in row[3:6]))
         for i in range(6, len(row)):
             out[out['sensor_names'][i-6]].append(row[i])
 
@@ -511,8 +511,8 @@ def horizon_weekly():
     # Important: this only works assuming that res_all is correctly sorted
 
     for row in res_all:
-        out['days'].append('-'.join(row[:3]))
-        out['times'].append(':'.join(row[3:6]))
+        out['days'].append('-'.join(format(y, '02d') for y in row[:3]))
+        out['times'].append(':'.join(format(y, '02d') for y in row[3:6]))
         for i in range(6, len(row)):
             out[out['sensor_names'][i-6]].append(row[i])
 
@@ -552,8 +552,8 @@ def horizon_daily():
     # Important: this only works assuming that res_all is correctly sorted
 
     for row in res_all:
-        out['days'].append('-'.join(row[:3]))
-        out['times'].append(':'.join(row[3:6]))
+        out['days'].append('-'.join(format(y, '02d') for y in row[:3]))
+        out['times'].append(':'.join(format(y, '02d') for y in row[3:6]))
         for i in range(6, len(row)):
             out[out['sensor_names'][i-6]].append(row[i])