First version of partial index---for [horizon yearly] only

py/import.py

@@ -70,13 +70,17 @@ for hus_name in next(os.walk(BASEDIR))[1]:
 				units = list(compress(units, not_empty))
 
 				# Create the table
-				col_names = ['Date', 'Time']
+				col_names = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'Second']
 				col_names.extend(f'{family_names[i]}.{sensor_names[i]}.{units[i]}' for i in range(3, len(family_names)))
 				#print(col_names)
 
 				cur.execute(f"CREATE TABLE {table_name}{*col_names,}")
 				con.commit()
 
+				cur.execute(f'CREATE INDEX idx_{card}_Year ON {table_name} (Year)')
+				cur.execute(f'CREATE INDEX idx_{card}_Hour ON {table_name} (Hour)')
+				con.commit()
+
 		#exit()
 
 		print(f'Processing {txt}...')
@@ -89,11 +93,16 @@ for hus_name in next(os.walk(BASEDIR))[1]:
 					# Skip two lines; we don't need them anymore
 					header1 = next(reader)
 					header2 = next(reader)
-				else:					
-					data.append(tuple(compress(line, not_empty)))
-					#print(data)
-					#exit()
-
+				else:
+					# First split the date into three columns (y,m,d)
+					row = line[0].split('-')
+					# Then split the time into three columns (h,m,s)
+					row.extend(line[1].split(':'))
+					# Then add the data (without the date, time, and empty columns)
+					row.extend(tuple(compress(line, not_empty))[2:])
+
+					data.append(row)
+					
 					# Sometimes, some of the lines in a file are the wrong size.
 					# When that happens, we skip the file.
 					if len(data[-1]) != len(col_names):
@@ -114,4 +123,3 @@ for hus_name in next(os.walk(BASEDIR))[1]:
 
 			cur.executemany(query, data)
 			con.commit()
-

py/main.py

@@ -11,16 +11,18 @@ import glob
 
 DB_NAMES = {
     'Charlie': 'Charlie.db',
-    'Pilgatan': 'Pilgatan.db',
+    #'Pilgatan': 'Pilgatan.db',
 }
 
 table_names = {
     'Charlie': ['Hus_Charlie_card_1', 'Hus_Charlie_card_4'],
-    'Pilgatan': ['Hus_Pilgatan_card_v1', 'Hus_Pilgatan_card_v6'],
+    #'Pilgatan': ['Hus_Pilgatan_card_v1', 'Hus_Pilgatan_card_v6'],
 }
 
-date_col = "Date"
-time_col = "Time"
+#date_col = "Date"
+date_cols = ['Year', 'Month', 'Day']
+#time_col = "Time"
+time_cols = ['Hour', 'Minute', 'Second']
 
 
 # Startup
@@ -44,7 +46,7 @@ for hus in DB_NAMES:
     TABLE_FAMILIES_SENSORS[hus] = {}
     for table_name, col_names in TABLE_COLS[hus].items():
         TABLE_FAMILIES_SENSORS[hus][table_name] = {}
-        for col_name in col_names[2:]:
+        for col_name in col_names[6:]:
             match = re.search(r'(.*)\.(.*)\..*', col_name)
             fam, sen = match[1], match[2]
             if fam not in TABLE_FAMILIES_SENSORS[hus][table_name]:
@@ -57,7 +59,7 @@ def get_table_cols(hus, fam, typ):
     for table_name in TABLE_FAMILIES_SENSORS[hus]:
         if fam in TABLE_FAMILIES_SENSORS[hus][table_name]:
             sen_cols = [col for col in TABLE_COLS[hus][table_name] if re.match(f'{fam}\.(.*)\.{typ}', col)]
-            col_names = [date_col, time_col]
+            col_names = [*date_cols, *time_cols]
             col_names.extend(sen_cols)
             return table_name, col_names
     return None
@@ -336,17 +338,17 @@ def horizon_yearly():
     sql_col_names = ','.join(f'"{x}"' for x in col_names)
 
     out = {}
-    out['sensor_names'] = col_names[2:]
+    out['sensor_names'] = col_names[6:]    
 
     aux_data = {}
     for sensor_name in out['sensor_names']:
         aux_data[sensor_name] = []
 
-    for hour in ['00','06','12','18']:        
+    for hour in ['00','06','12','18']:
         query  = f'SELECT {sql_col_names} FROM "{table_name}"'
-        query += f'WHERE "{date_col}" LIKE "{yea}-%"'
-        query += f'AND "{time_col}" LIKE "{hour}:%"'
-        query += f' GROUP BY "{date_col}"'
+        query += f' WHERE "Year" = "{yea}"'
+        query += f' AND "Hour" = "{hour}"'
+        query += f' GROUP BY "Month", "Day"'
         query += ';'
 
         cur = conn.execute(query)
@@ -354,14 +356,14 @@ def horizon_yearly():
 
         for row in res_all:
             for i, sensor_name in enumerate(out['sensor_names']):
-                aux_data[sensor_name].append((row[0], row[1], row[i+2]))
+                aux_data[sensor_name].append((*row[:6], row[i+6]))
 
     for i, sensor_name in enumerate(out['sensor_names']):
-        sensor_data_sorted = sorted(aux_data[sensor_name])
+        sensor_data_sorted = sorted(aux_data[sensor_name])        
         if i == 0:
-            out['days'] = [x[0] for x in sensor_data_sorted]
-            out['times'] = [x[1] for x in sensor_data_sorted]
-        out[sensor_name] = [x[2] for x in sensor_data_sorted]
+            out['days'] = ['-'.join(x[:3]) for x in sensor_data_sorted]
+            out['times'] = [':'.join(x[3:6]) for x in sensor_data_sorted]
+        out[sensor_name] = [x[6] for x in sensor_data_sorted]
 
     return out