TimberVis Flexiboard: Visualization and Exploration Flexiboard for Timber Buildings IoT data sensors. The pulse of the wood Monitoring of Wooden houses: Time series of sensors data measuring humidity, temperatures, vibrations and weather conditions. https://lnu.se/forskning/forskningsprojekt/projekt-flexiboard-for-visualisering-och-utforskning-av-trabyggnader/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
infravis-trahust/py/main.py

608 lines
19 KiB

import itertools
from flask import Flask, request
import sqlite3
from datetime import date, timedelta
from flask_cors import CORS
import re
import json
import os
import glob
DB_NAMES = {
'Charlie': 'Charlie.db',
#'Pilgatan': 'Pilgatan.db',
}
table_names = {
'Charlie': ['Hus_Charlie_card_1', 'Hus_Charlie_card_4'],
#'Pilgatan': ['Hus_Pilgatan_card_v1', 'Hus_Pilgatan_card_v6'],
}
#date_col = "Date"
date_cols = ['Year', 'Month', 'Day']
#time_col = "Time"
time_cols = ['Hour', 'Minute', 'Second']
# Startup
app = Flask(__name__)
CORS(app)
TABLE_COLS = {}
TABLE_FAMILIES_SENSORS = {}
for hus in DB_NAMES:
conn = sqlite3.connect(DB_NAMES[hus])
# Prepare the auxiliary metadata
TABLE_COLS[hus] = {}
for table_name in table_names[hus]:
cur = conn.execute(f"PRAGMA table_info('{table_name}')")
res = cur.fetchall()
cur.close()
TABLE_COLS[hus][table_name] = [x[1] for x in res]
TABLE_FAMILIES_SENSORS[hus] = {}
for table_name, col_names in TABLE_COLS[hus].items():
TABLE_FAMILIES_SENSORS[hus][table_name] = {}
for col_name in col_names[6:]:
match = re.search(r'(.*)\.(.*)\..*', col_name)
fam, sen = match[1], match[2]
if fam not in TABLE_FAMILIES_SENSORS[hus][table_name]:
TABLE_FAMILIES_SENSORS[hus][table_name][fam] = []
TABLE_FAMILIES_SENSORS[hus][table_name][fam].append(sen)
def get_table_cols(hus, fam, typ):
# Get the right sensors for the family
for table_name in TABLE_FAMILIES_SENSORS[hus]:
if fam in TABLE_FAMILIES_SENSORS[hus][table_name]:
sen_cols = [col for col in TABLE_COLS[hus][table_name] if re.match(f'{fam}\.(.*)\.{typ}', col)]
col_names = [*date_cols, *time_cols]
col_names.extend(sen_cols)
return table_name, col_names
return None
def get_table_col_fam(hus, sen, typ):
for table_name in TABLE_FAMILIES_SENSORS[hus]:
for col in TABLE_COLS[hus][table_name]:
#print(table_name, col)
match = re.match(f'(.*)\.{sen}\.{typ}', col)
if match:
return table_name, col, match[1]
return None
@app.route("/")
def root():
return TABLE_FAMILIES_SENSORS
@app.route("/parallel/daily")
def parallel_daily():
# Process parameters
hus = request.args.get('hus', 'Charlie')
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
day = request.args.get('day', '2023-01-01')
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
# Split date
yea, mon, day = day.split('-')
query = (f'SELECT {sql_col_names} FROM {table_name}'
f' WHERE Year=? AND Month=? AND Day=?'
f' GROUP BY Hour;')
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, [yea, mon, day])
res_all = cur.fetchall()
cur.close()
sample = [["pos1", *[x*10+10 for x in range(len(col_names)-6)]]]
myxs = {}
for data_point in res_all:
point_id = ':'.join(data_point[3:6])
myxs[point_id] = 'pos1'
data_point = [point_id, *data_point[6:]]
sample.append(data_point)
return dict(sample=sample, myxs=myxs)
@app.route("/parallel/weekly")
def parallel_weekly():
# Process parameters
hus = request.args.get('hus', 'Charlie')
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
wee = int(request.args.get('week', '1'))
yea = int(request.args.get('year', '2023'))
hou = int(request.args.get('hour', '0'))
# Make sure format is right
hou = f'{hou:02d}'
#if not wee or not yea:
# return 'ERROR: You need to at least specify the parameters "week" and "year".'
# Find monday from the given week
monday = date.fromisocalendar(yea, wee, 1)
weekday_names = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
weekdays = {str(monday + timedelta(days=x)):weekday_names[x] for x in range(7)}
#print(weekdays)
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
# Complex query to get each day of the week
query = f'SELECT {sql_col_names} FROM {table_name} WHERE ('
query += 'OR'.join(['(Year=? AND Month=? AND Day=?)'] * len(weekdays))
query += ') AND Hour=?'
query += ' GROUP BY Day;'
# Break each date into three components then merge them all (in a flat list)
params = [y for d in list(weekdays.keys()) for y in d.split('-')]
params.append(hou)
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, params)
res_all = cur.fetchall()
sample = [["pos1", *[x*10+10 for x in range(len(col_names)-6)]]]
myxs = {}
# group by day; take first of each group
for i, data_point in enumerate(res_all):
wd = list(weekdays.items())[i]
point_id = f'{wd[1]} ({wd[0]})'
myxs[point_id] = 'pos1'
data_point = [point_id, *data_point[6:]]
sample.append(data_point)
return dict(sample=sample, myxs=myxs)
@app.route("/parallel/monthly")
def parallel_monthly():
# Process parameters
hus = request.args.get('hus', 'Charlie')
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
mon = int(request.args.get('month', '1'))
yea = int(request.args.get('year', '2023'))
wda = int(request.args.get('weekday', 1))
hou = int(request.args.get('hour', '0'))
# Make sure format is right
hou = f'{hou:02d}'
#if not wee or not yea:
# return 'ERROR: You need to at least specify the parameters "week" and "year".'
# first days of the 4 weeks
first = date(yea, mon, wda)
days = [str(first + timedelta(days=7*x)) for x in range(4)]
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
# Complex query to get each day separately
query = f'SELECT {sql_col_names} FROM {table_name} WHERE ('
query += 'OR'.join(['(Year=? AND Month=? AND Day=?)'] * len(days))
query += ') AND Hour=?'
query += ' GROUP BY Day;'
# Break each date into three components then merge them all (in a flat list)
params = [y for d in days for y in d.split('-')]
params.append(hou)
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, params)
res_all = cur.fetchall()
sample = [["pos1", *[x*10+10 for x in range(len(col_names)-6)]]]
myxs = {}
# group by day; take first of each group
for data_point in res_all:
point_id = '-'.join(data_point[:3])
myxs[point_id] = 'pos1'
data_point = [point_id, *data_point[6:]]
sample.append(data_point)
return dict(sample=sample, myxs=myxs)
@app.route("/parallel/yearly")
def parallel_yearly():
# Process parameters
hus = request.args.get('hus', 'Charlie')
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
yea = request.args.get('year', '2023')
day = int(request.args.get('day', '1'))
hou = int(request.args.get('hour', '0'))
# Make sure format is right
day = f'{day:02d}'
hou = f'{hou:02d}'
#if not wee or not yea:
# return 'ERROR: You need to at least specify the parameters "week" and "year".'
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(f'SELECT {sql_col_names} FROM {table_name}'
' WHERE Year=? AND Day=? AND Hour=? GROUP BY Month;', [yea, day, hou])
res_all = cur.fetchall()
sample = [["pos1", *[x*10+10 for x in range(len(col_names)-6)]]]
myxs = {}
month_names = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
# group by day; take first of each group
for data_point in res_all:
month_num = int(data_point[1])-1
point_id = f'{month_names[month_num]}'
myxs[point_id] = 'pos1'
data_point = [point_id, *data_point[6:]]
sample.append(data_point)
return dict(sample=sample, myxs=myxs)
@app.route("/grid/yearly")
def grid_yearly():
hus = request.args.get('hus', 'Charlie')
# Process parameters
sen = request.args.get('sensor', 'Temp_MP1_1_Pos1')
typ = request.args.get('type', 'celsius')
yea = request.args.get('year', '2023')
hou = int(request.args.get('hour', '0'))
# Make sure format is right
hou = f'{hou:02d}'
# Get the columns for the sensor
table_name, col_name, fam = get_table_col_fam(hus, sen, typ)
col_names = [*date_cols, *time_cols, col_name]
sql_col_names = ','.join(f'"{x}"' for x in col_names)
query = f'SELECT {sql_col_names} FROM {table_name}'
query += f' WHERE Year=? AND Hour=?'
query += f' GROUP BY Year,Month,Day;'
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, [yea, hou])
res_all = cur.fetchall()
out = []
out.append(['-'.join(x[:3]) for x in res_all])
out.append([':'.join(x[3:6]) for x in res_all])
out.append([x[6] for x in res_all])
return out
@app.route("/grid/monthly")
def grid_monthly():
hus = request.args.get('hus', 'Charlie')
# Process parameters
sen = request.args.get('sensor', 'Temp_MP1_1_Pos1')
typ = request.args.get('type', 'celsius')
yea = request.args.get('year', '2023')
mon = int(request.args.get('month', '1'))
hou = int(request.args.get('hour', '0'))
# Make sure format is right
hou = f'{hou:02d}'
mon = f'{mon:02d}'
# Get the columns for the sensor
table_name, col_name, fam = get_table_col_fam(hus, sen, typ)
col_names = [*date_cols, *time_cols, col_name]
sql_col_names = ','.join(f'"{x}"' for x in col_names)
query = f'SELECT {sql_col_names} FROM {table_name}'
query += f' WHERE Year=? AND Month=? AND Hour=?'
query += f' GROUP BY Year,Month,Day;'
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, [yea, mon, hou])
res_all = cur.fetchall()
out = []
out.append(['-'.join(x[:3]) for x in res_all])
out.append([':'.join(x[3:6]) for x in res_all])
out.append([x[6] for x in res_all])
return out
@app.route("/grid/weekly")
def grid_weekly():
hus = request.args.get('hus', 'Charlie')
# Process parameters
sen = request.args.get('sensor', 'Temp_MP1_1_Pos1')
typ = request.args.get('type', 'celsius')
yea = int(request.args.get('year', '2023'))
wee = int(request.args.get('week', '1'))
hou = int(request.args.get('hour', '0'))
# Make sure format is right
hou = f'{hou:02d}'
monday = date.fromisocalendar(yea, wee, 1)
weekday_names = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
weekdays = {str(monday + timedelta(days=x)):weekday_names[x] for x in range(7)}
#print(weekdays)
# Get the columns for the sensor
table_name, col_name, fam = get_table_col_fam(hus, sen, typ)
col_names = [*date_cols, *time_cols, col_name]
sql_col_names = ','.join(f'"{x}"' for x in col_names)
# Complex query to get each day separately
query = f'SELECT {sql_col_names} FROM {table_name} WHERE ('
query += 'OR'.join(['(Year=? AND Month=? AND Day=?)'] * len(weekdays))
query += ') AND Hour=?'
query += f' GROUP BY Year,Month,Day;'
# Break each date into three components then merge them all (in a flat list)
params = [y for d in list(weekdays.keys()) for y in d.split('-')]
params.append(hou)
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, params)
res_all = cur.fetchall()
out = []
out.append(['-'.join(x[:3]) for x in res_all])
out.append([':'.join(x[3:6]) for x in res_all])
out.append([x[6] for x in res_all])
return out
@app.route("/horizon/yearly")
def horizon_yearly():
hus = request.args.get('hus', 'Charlie')
# Process parameters
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
yea = int(request.args.get('year', '2023'))
conn = sqlite3.connect(DB_NAMES[hus])
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
# Important: this code is outdated when compared to the other horizon endpoints
out = {}
out['sensor_names'] = col_names[6:]
aux_data = {}
for sensor_name in out['sensor_names']:
aux_data[sensor_name] = []
for hour in ['00','06','12','18']:
query = f'SELECT {sql_col_names} FROM "{table_name}"'
query += f' WHERE "Year" = "{yea}"'
query += f' AND "Hour" = "{hour}"'
query += f' GROUP BY "Month", "Day"'
query += ';'
cur = conn.execute(query)
res_all = cur.fetchall()
for row in res_all:
for i, sensor_name in enumerate(out['sensor_names']):
aux_data[sensor_name].append((*row[:6], row[i+6]))
for i, sensor_name in enumerate(out['sensor_names']):
sensor_data_sorted = sorted(aux_data[sensor_name])
if i == 0:
out['days'] = ['-'.join(x[:3]) for x in sensor_data_sorted]
out['times'] = [':'.join(x[3:6]) for x in sensor_data_sorted]
out[sensor_name] = [x[6] for x in sensor_data_sorted]
return out
@app.route("/horizon/monthly")
def horizon_monthly():
hus = request.args.get('hus', 'Charlie')
# Process parameters
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
yea = request.args.get('year', '2023')
mon = int(request.args.get('month', '1'))
# Make sure the format is right
mon = f'{mon:02d}'
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
#print(sql_col_names)
out = {}
out['sensor_names'] = col_names[6:]
out['days'] = []
out['times'] = []
for s in out['sensor_names']:
out[s] = []
query = (f'SELECT {sql_col_names} FROM {table_name}'
f' WHERE Year=? AND Month=?'
f' GROUP BY Day, Hour')
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, [yea, mon])
res_all = cur.fetchall()
# Important: this only works assuming that res_all is correctly sorted
for row in res_all:
out['days'].append('-'.join(row[:3]))
out['times'].append(':'.join(row[3:6]))
for i in range(6, len(row)):
out[out['sensor_names'][i-6]].append(row[i])
return out
@app.route("/horizon/weekly")
def horizon_weekly():
hus = request.args.get('hus', 'Charlie')
# Process parameters
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
yea = int(request.args.get('year', '2023'))
wee = int(request.args.get('week', '1'))
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
monday = date.fromisocalendar(yea, wee, 1)
weekday_names = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
weekdays = {str(monday + timedelta(days=x)):weekday_names[x] for x in range(7)}
#print(weekdays)
out = {}
out['sensor_names'] = col_names[6:]
out['days'] = []
out['times'] = []
for s in out['sensor_names']:
out[s] = []
query = f'SELECT {sql_col_names} FROM {table_name} WHERE '
query += 'OR'.join(['(Year=? AND Month=? AND Day=?)'] * len(weekdays))
query += ' GROUP BY Year, Month, Day, Hour'
# Break each date into three components then merge them all (in a flat list)
params = [y for d in list(weekdays.keys()) for y in d.split('-')]
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, params)
res_all = cur.fetchall()
# Important: this only works assuming that res_all is correctly sorted
for row in res_all:
out['days'].append('-'.join(row[:3]))
out['times'].append(':'.join(row[3:6]))
for i in range(6, len(row)):
out[out['sensor_names'][i-6]].append(row[i])
return out
@app.route("/horizon/daily")
def horizon_daily():
hus = request.args.get('hus', 'Charlie')
# Process parameters
fam = request.args.get('family', 'MP1_1')
typ = request.args.get('type', 'celsius')
day = request.args.get('day', '2023-01-01')
yea, mon, day = day.split('-')
# Get the right sensors for the family
table_name, col_names = get_table_cols(hus, fam, typ)
sql_col_names = ','.join(f'"{x}"' for x in col_names)
out = {}
out['sensor_names'] = col_names[6:]
out['days'] = []
out['times'] = []
for s in out['sensor_names']:
out[s] = []
query = f'SELECT {sql_col_names} FROM "{table_name}"'
query += f'WHERE Year=? AND Month=? AND Day=?;'
conn = sqlite3.connect(DB_NAMES[hus])
cur = conn.execute(query, [yea, mon, day])
res_all = cur.fetchall()
# Important: this only works assuming that res_all is correctly sorted
for row in res_all:
out['days'].append('-'.join(row[:3]))
out['times'].append(':'.join(row[3:6]))
for i in range(6, len(row)):
out[out['sensor_names'][i-6]].append(row[i])
return out
@app.route('/boards')
def boards():
hus = request.args.get('hus', 'Charlie')
all_files = glob.glob(f"Boards/{hus}/*.json")
output = []
for filename in all_files:
with open(filename, 'r') as f:
_id = os.path.basename(filename)
obj = json.load(f)
output.append(dict(Filename=_id, Name=obj['Name'], Desc=obj['Description']))
return output
@app.route('/boards/save', methods=['POST'])
def boards_save():
b = request.json
if not b:
return "Failed; no board was sent via POST."
hus = b['Hus']
try:
os.mkdir(f'Boards/{hus}')
except FileExistsError:
# No problem
pass
_id = re.sub('[^0-9a-zA-Z]', '-', b['Name'])
with open(f'Boards/{hus}/{_id}.json', 'w') as f:
json.dump(b, f)
return 'OK'
@app.route('/boards/load')
def boards_load():
hus = request.args.get('hus')
filename = request.args.get('filename')
if not filename or not hus:
return "ERROR: the parameters 'hus' and 'filename' are both mandatory."
fullpath = f'Boards/{hus}/{filename}'
try:
with open(fullpath, 'r') as f:
output = json.load(f)
except FileNotFoundError:
return "ERROR: specified hus/file was not found."
return output