meter_sim/download_thingsboard.py

# -*- coding: utf-8 -*-
"""
Created on Fri Aug 14 21:10:25 2020

@author: solo
"""

import requests
import json
import time
import pandas as pd
import sys
import pandas as pd
import os
from datetime import datetime

src_server = 'http://124.222.45.156:18080'
src_req_url = '%s/api/auth/login'%(src_server)
src_username = "cz_dld@thingsboard.org"
src_password = "tenant"
src_s = json.dumps({"username":src_username, "password":src_password})
src_r = requests.post(src_req_url, headers={'Accept': 'application/json'}, data=src_s)
src_rj = json.loads(src_r.text.encode())
print('src rj:' + str(src_rj))

startdate='2024-1-18'
enddate='2024-1-19'
devidx = 1
src_deviceid='d203df30-af8e-11ee-9972-3dd738949819'

keys=[]


keys.append({'src':'es_ap','tgt':'es_ap','intv':1})

keys.append({'src':'load_ap','tgt':'load_ap','intv':1})

'''
# cell voltages
for i in range(5):
    for j in range(52):
        keys.append({'src':'bms1_p%d_v%d'%(i+1,j+1),'tgt':'bms1_p%d_v%d'%(i+1,j+1),'intv':60}) 
for i in range(5):
    for j in range(52):
        keys.append({'src':'bms1_p%d_t%d'%(i+1,j+1),'tgt':'bms1_p%d_t%d'%(i+1,j+1),'intv':180})  

#  fa 1 - 5  ############################################################################
for i in range(5):
    keys=keys+[
            {'src':'fa%d_co'%(i+1),'tgt':'fa%d_co'%(i+1),'intv':60*5},
            {'src':'fa%d_fire'%(i+1),'tgt':'fa%d_fire'%(i+1),'intv':60*5},
            {'src':'fa%d_smoke'%(i+1),'tgt':'fa%d_smoke'%(i+1),'intv':60*5},
            {'src':'fa%d_status'%(i+1),'tgt':'fa%d_status'%(i+1),'intv':60*5},
            {'src':'fa%d_t1'%(i+1),'tgt':'fa%d_t1'%(i+1),'intv':60*5},
            {'src':'fa%d_t2'%(i+1),'tgt':'fa%d_t2'%(i+1),'intv':60*5},
            {'src':'fa%d_voc'%(i+1),'tgt':'fa%d_voc'%(i+1),'intv':60*5},
          ]
'''

for key in keys:
    drange = pd.date_range(startdate,enddate)
    for date in drange:
        time.sleep(0.1)
        devpath='data/dev%d'%(devidx)
        if not os.path.exists(devpath):
            os.mkdir(devpath)
        keypath='%s/%s'%(devpath,key['tgt'])
        if not os.path.exists(keypath):
            os.mkdir(keypath)
        try:
            print('getting data for src:',key['src'],'tgt:',key['tgt'],'date:','%04d-%02d-%02d'%(date.year, date.month, date.day))
            startTs = int(time.mktime(time.strptime('%04d-%02d-%02d 00:00:00'%(date.year, date.month, date.day), "%Y-%m-%d %H:%M:%S"))*1000)
            endTs = int(time.mktime(time.strptime('%04d-%02d-%02d 23:59:59'%(date.year, date.month, date.day), "%Y-%m-%d %H:%M:%S"))*1000)        
            req_url = '%s/api/plugins/telemetry/DEVICE/%s/values/timeseries?keys=%s&startTs=%d&endTs=%d&limit=50000&agg=NONE'%(src_server,src_deviceid,key['src'],startTs,endTs)
            r = requests.get(req_url, headers={'Accept': 'application/json','X-Authorization':'Bearer %s'%src_rj['token']})
            if r.status_code != 200:
                print(r)
                continue
            rj_val = json.loads(r.text.encode())
            l = rj_val[key['src']]
            last_ts = 0
            ts=[]
            val=[]
            l.reverse()
            for itm in l:
                if abs(itm['ts'] - last_ts) > (key['intv']-5)*1000:
                    last_ts = itm['ts']
                    ts.append(datetime.fromtimestamp(itm['ts']/1000).strftime("%H:%M:%S"))
                    val.append(itm['value'])
            #df = pd.DataFrame(data = [ts,val], columns=['ts',key])
            #df.to_excel('com_ap_%04d-%02d-%02d.xls'%(date.year, date.month, date.day))
            df = pd.DataFrame()
            df['ts']=ts
            df[key['tgt']]=val
            df.to_csv('%s/%04d-%02d-%02d.csv'%(keypath,date.year, date.month, date.day),index=None)
            print('saved')
            time.sleep(0.02)
        except:
            print("get data error @"+str(date)+' for key:%s'%(key['src']))
            #sys.exit()
            continue
        else:
            continue
print('done')