import errno
import json
import subprocess
import pandas as pd
import os
from os import listdir
from os.path import isfile, join
import datetime
import antgen
import matplotlib.pyplot as plt
[docs]class ConsumptionGenerator:
def __init__(self, configuration_file, path_steps_seconds, path_steps_minutes):
"""
This class generates the consumption profiles of the community houses and calculates the aggregate consumption
Args:
configuration_file: path of the JSON community configuration file
path_steps_seconds: path of the consumption profiles generated by ANTGen (at 1 Hz)
path_steps_minutes: path of the resampled consumption profiles (at 1/60Hz)
"""
self.community = None
self.configuration_file = configuration_file
self.path_steps_seconds = path_steps_seconds
self.path_steps_minutes = path_steps_minutes
[docs] def datetime_range(self, start, end, delta):
"""
Generates a list between start and end times, with a specific step
Args:
start: start time
end: end time
delta: step between start and end
"""
current = start
while current < end:
yield current
current += delta
[docs] def date_ranges_overlap(self, start1, end1, start2, end2):
"""
Checks if two date ranges overlap
Args:
start1: Start of first date
end1: End of first date
start2: Start of second date
end2: End of second date
Returns:
true if the date ranges overlap, and false otherwise
"""
return (start1 <= end2) and (end1 >= start2)
[docs] def show_house_graph(self, house_num):
"""
Plots the total consumption of a specific house
Args:
house_num: number of the house to be plotted
"""
width = 0.35 # the width of the bars: can also be len(x) sequence
labels = list(range(0, 24))
fig, ax = plt.subplots()
#Energy of the community
#energy = pd.read_csv('output/minute/energy.csv', sep=';')["Power"]
#ax.bar(labels, energy, width, label='Energy')
df = pd.read_csv(self.path_steps_minutes + "/house" + house_num + '/total.csv', sep=';')
df['Date'] = pd.to_datetime(df['Date'])
df = df.set_index('Date').resample('60min').mean()
print(df);
ax.plot(labels, df, width)
plt.xticks(labels)
ax.set_ylabel('Power (W)')
ax.set_title('Hour')
ax.legend()
#plt.axhline(y=contractedPower, linewidth=1, color='k')
plt.show()
[docs] def calculate_max_of_a_timeslot(self, start_date, end_date, appliance, house):
"""
Calculates the maximum power of an activity based on the consumption profile
Args:
start_date: start date of the activity (when the appliance is being used)
end_date: end date of the activity (when the appliance stops being used)
appliance: name of the appliance (corresponds to the name of the appliance folder of the dataset)
house: number of the house of the activity
Returns:
maximum power achieved by the appliance in this specific activity
"""
#Get power of the appliance in order to calculate the max between the start and end of the timeslot
df = pd.read_csv(self.path_steps_minutes + "/house" + str(house) + '/' + appliance + ".csv", sep=';') # Header=None to indicate that the first row is data and not colummn names
df.columns = ['Date', 'Power']
df = df[:24 * 60 * 60] # Only the first day is important (24 hours * 60 minutes * 60 seconds)
#Convert the start and end strings to dates
start_obj = datetime.datetime.strptime(start_date, '%Y-%m-%d %H:%M:%S') # Convert string to datetime object
end_obj = datetime.datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S') # Convert string to datetime object
obj = start_obj
#Variable to save the max value
max = 0
#Get the power of each minute of the appliance between the start and end
while(obj != end_obj):
if (float(df[df.Date == str(obj)]["Power"]) > max):
max = float(df[df.Date == str(obj)]["Power"])
obj = obj + datetime.timedelta(minutes=1) #Next minute
return max
[docs] def calculate_contracted_power(self, community):
"""
Calculates the contracted power of the community by summing all the contracted powers of each house presented in the json configuration file.
Args:
community: community information (read from the JSON file)
Returns:
contracted power of the whole community
"""
print("Calculating the contracted power of the community");
contracted_power = 0.0;
num_houses = len(community)
for house in range(num_houses):
contracted_power += (float(community[house]["contracted_power"])*1000)
return contracted_power
[docs] def get_timeslots(self, community, is_flexible):
"""
Gets all the activities of the community (i.e. of all houses of the community).
Based on the consumption files of each house, it extracts the activities done (start, end, duration, max_power, power, appliance and house)
Args:
community: community information (read from the JSON file)
is_flexible: if True, only the activities of the flexible appliances are extracted (Dishwasher, Vaccum Cleaner, Washing Machine, Dryer, Iron and Cooking Stove), otherwise all the appliances are extracted (Amplifier, Breadcutter, CD Player, Coffe Maker, Cooking Stove, Dishwasher, Dryer, Freezer, Iron, Kettle, Microwave, PC, Printer, Refrigerator, Toaster, TV, Vaccum Cleaner and Washing Machine)
Returns:
array containing all the activities
"""
print("Creating the Timeslots of the community")
num_houses = len(community)
if (is_flexible):
flexible_appliances = ['DISHWASHER', 'VACUUMCLEANER', 'WASHINGMACHINE', 'DRYER', 'IRON', 'COOKINGSTOVE']
appliances = flexible_appliances
else:
appliances = ['AMPLIFIER', 'BREADCUTTER', 'CDPLAYER', 'COFFEEMAKER', 'COOKINGSTOVE', 'DISHWASHER', 'DRYER', 'FREEZER', 'IRON', 'KETTLE', 'MICROWAVE', 'PC', 'PRINTER', 'REFRIGERATOR', 'TOASTER', 'TV', 'VACUUMCLEANER', 'WASHINGMACHINE']
all_timeslots = []
timeslots_array = []
for z in range(num_houses):
house_timeslots = []
for appliance in appliances:
applianceTimeslots = []
df = pd.read_csv(self.path_steps_minutes + "/house" + str(z) + '/' + appliance + ".csv", sep=';') # Header=None to indicate that the first row is data and not colummn names
df.columns = ['Date', 'Power']
df = df[:24 * 60 * 60] # Only the first day is important (24 hours * 60 minutes * 60 seconds)
used_appliances_rows = df[df.Power != 0.0] # Saves the records which power > 0 (times of the day where the appliance is used)
for index, row in used_appliances_rows.iterrows():
if not df.empty:
date_time_obj = datetime.datetime.strptime(row["Date"], '%Y-%m-%d %H:%M:%S') #Convert string to datetime object
next_minute = date_time_obj + datetime.timedelta(minutes=1) # Sums one minute (to get the power of next minute - in order to see if the appliance is used in the next minute or not) - to verify if it is the end date or not
previous_minute = date_time_obj - datetime.timedelta(minutes=1) # Subtracts one minute (to get the power of previous minute - in order to see if the appliance is used in the previous minute or not) - to verify if it is the start date or not
power_next_minute = df[df.Date == str(next_minute)]["Power"]
power_previous_minute = df[df.Date == str(previous_minute)]["Power"]
try:
if (float(power_previous_minute) == 0):
start_date = str(date_time_obj)
except TypeError:
print("")
try:
if (float(power_next_minute) == 0):
#end_date = str(df[df.Date == str(end_date)]["Date"])
end_date = str(date_time_obj)
duration = datetime.datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S') - datetime.datetime.strptime(start_date, '%Y-%m-%d %H:%M:%S');
durationInMinutes = duration.seconds // 60 % 60
#print("Time-slot created: " + "\nStart: " + start_date + "\nEnd: " + end_date + "\nDuration: " + str(durationInMinutes) + " minutes\nPower: " + str(row['Power']) + "\nAppliance: " + appliance + "\nHouse: " + str(z) + "\n")
timeslots_array.append(
{"Start": start_date, "End": end_date, "Duration": str(durationInMinutes), "max_power": self.calculate_max_of_a_timeslot(start_date, end_date, appliance, z), "Power": row['Power'],
"Appliance": appliance, "House": z})
applianceTimeslots.append(
{"Start": start_date, "End": end_date, "Duration": str(durationInMinutes), "max_power": self.calculate_max_of_a_timeslot(start_date, end_date, appliance, z), "Power": row['Power'], "Appliance": appliance, "House": z})
except TypeError:
print("")
house_timeslots.append({"Appliance": appliance, "Timeslots": house_timeslots})
all_timeslots.append({"House": str(z), "Appliances": house_timeslots})
return timeslots_array
[docs] def create_houses_files(self, community):
"""
Creates house files with the extension ".conf" using ANTGen based on community configuration file.
The files are generated and saved in the houses folder (e.g. house4).
Args:
community: community information (read from the JSON file)
"""
print("Creating simulation files for each house")
num_houses = len(community)
# Create the houses folder to store the user configuration files (if it does not exist)
if not os.path.exists("houses"):
os.mkdir("houses")
# for every house of the community
for z in range(num_houses):
print("Creating simulation file for house " + str(z))
f = open("houses/house" + str(z) + ".conf", "w")
f.write("[GENERAL]" + "\n")
f.write("name = Sample configuration file" + "\n")
#f.write("seed = 1618407485" + "\n")
#f.write("days = 1" + "\n")
f.write("\n")
f.write("[users]" + "\n")
for y in range(len(community[z]["people"])):
f.write(str(community[z]["people"][y]) + " = house" + str(z) + "_" + str(community[z]["people"][y]) + ".conf" + "\n")
f.close()
[docs] def create_users_files(self, community):
"""
Creates users files with the extension ".conf" using ANTGen based on community configuration file.
The files are generated and saved in the users folder (e.g. house2_Ann.conf).
Args:
community: community information (read from the JSON file)
"""
print("Creating simulation files for each user")
num_houses = len(community)
# Create the users folder to store the user configuration files (if it does not exist)
if not os.path.exists("users"):
os.mkdir("users")
# for every house of the community
for z in range(num_houses):
print("Creating simulation file for users of house " + str(z))
# for every person of the house (gets its schedule and creates a new file)
for y in range(len(community[z]["people"])):
print("Creating simulation file for user " + str(community[z]["people"][y]))
f = open("users/house" + str(z) + "_" + str(community[z]["people"][y]) + ".conf", "w")
f.write("[GENERAL]" + "\n")
f.write("name = " + str(community[z]["people"][y]) + "\n\n")
f.write("[presence]" + "\n")
# for every day of the week (monday, tuesday, wednesday, etc)
for x in range(len(community[z]["schedules"][y]["presence"])):
f.write(str(community[z]["schedules"][y]["presence"][x]["day"]) + " = " + (str(community[z]["schedules"][y]["presence"][x]["schedule"])).replace("]", "").replace("[", "").replace("'", "") + "\n")
for x in range(len(community[z]["schedules"][y]["activities"])):
f.write("\n[" + str(community[z]["schedules"][y]["activities"][x]["activity"]) + "]\n")
f.write("model = " + str(community[z]["schedules"][y]["activities"][x]["model"]) + "\n")
f.write("daily_runs = " + str(community[z]["schedules"][y]["activities"][x]["daily_runs"]) + "\n")
for w in range(len(community[z]["schedules"][y]["activities"][x]["schedule"])):
#print(community[z]["schedules"][y]["activities"][x]["schedule"][w])
f.write(str(community[z]["schedules"][y]["activities"][x]["schedule"][w]["day"]) + " = " + (str(community[z]["schedules"][y]["activities"][x]["schedule"][w]["schedule"])).replace("]", "").replace("[", "").replace("'", "") + "\n")
f.close()
[docs] def create_consumption_profiles(self, community, days, config_files_path):
"""
Creates the consumption profiles based on the house and users files (.conf).
The files are stored with a frequency of 1Hz and a folder for each house will be created (e.g. folders house0, house1, house2, etc)
Args:
community: community information (read from the JSON file)
days: number of days to generate data
config_files_path: path/folder where there are the house configuration files (.conf)
"""
antgen_path = str(antgen.__path__).split('[')[1].split("]")[0].replace("'", "")
print("Antgen Path: " + antgen_path)
# Update mapping.conf file based on antgen path
with open(antgen_path + "/mapping.conf", 'r') as file:
# Read the mapping.conf file
data = file.read()
# Replace path
data = data.replace("C:/Users/Nuno.Velosa.CORP/OneDrive - Unipartner IT Services, S.A/Desktop/antgen/antgen", antgen_path)
# Update the file
with open(antgen_path + "/mapping.conf", 'w') as file:
file.write(data)
print("Creating consumption profiles according to the houses files")
num_houses = len(community)
for x in range(num_houses):
# Create a folder for each community house in order to store their consumption profiles
if not os.path.exists(self.path_steps_seconds + "/output/house" + str(x)):
os.mkdir(self.path_steps_seconds + "/output/house" + str(x))
print("Creating consumption profile of house " + str(x))
baseload = "C" + str(int(community[x]["contracted_power"]*1000*0.03));
#subprocess.call(["python3", "/Users/nunovelosa/Desktop/antgen/main.py", "-n", baseload, "-d", "1", "-o", "output/house" + str(x), "-w", "-m", "/Users/nunovelosa/Desktop/antgen/mapping.conf", "-p", "houses/house" + str(x) + ".conf"])
subprocess.call(["python3", antgen_path + "/main.py", "-n", baseload, "-d", days, "-o", self.path_steps_seconds + "/output/house" + str(x), "-w", "-m", antgen_path + "/mapping.conf", "-p", config_files_path + "/house" + str(x) + ".conf"])
#print(total)
[docs] def convert_data_to_steps_of_one_minute(self, community, just_a_day = False, database_files = False):
"""
Resample the consumption profiles from 1Hz (1 value per second) to 1/60Hz (1 value per minute)
Args:
community: community information (read from the JSON file)
just_a_day: if true, only 1 day is considered, otherwise all the days are considered
database_files: if true, converts the database files, otherwise convertes the community files (false should be used)
"""
print("Resampling the data in order to get steps of 1 minute instead of 1 second.")
num_houses = len(community)
if database_files == False:
total = range(num_houses)
else:
total = range(1, 11)
# Create minutes folder
if not os.path.exists(self.path_steps_minutes):
os.mkdir(self.path_steps_minutes)
for z in total:
print("Resampling the files of HOUSE " + str(z))
if not os.path.exists(self.path_steps_minutes + "/house" + str(z)):
os.mkdir(self.path_steps_minutes + "/house" + str(z))
# All the files of the output (to convert from steps of a second to steps of a minute)
files_to_convert = [f for f in listdir(self.path_steps_seconds + "/output/house" + str(z)) if isfile(join(self.path_steps_seconds + "/output/house" + str(z), f))]
for file in files_to_convert:
if (file != "events.csv"):
print("Resampling the data of file " + file + " of house " + str(z))
df = pd.read_csv(self.path_steps_seconds + "/output/house" + str(z) + '/' + file, header=None,
sep=';') # Header=None to indicate that the first row is data and not colummn names
df.columns = ['Date', 'Power']
if just_a_day == True:
df = df[:24 * 60 * 60] # Only the first day is important (24 hours * 60 minutes * 60 seconds)
data = []
numberOfRows = int(len(df) / 60) # Convert 24*60*60 = 86400 seconds (rows) to 24*60 = 1440 minutes (rows)
for x in range(numberOfRows):
first = x * 60
last = first + 60
# print(str(first) + " " + str(last))
date = df.iloc[first]["Date"]
power = df[:last]["Power"].iloc[first:].mean() # Starts in first and ends in last, and calculates the average of the Power
data.append([date, power])
# Creates csv file in output/minute folder
output_directory = os.path.join(self.path_steps_minutes + "/house" + str(z))
outname = os.path.join(output_directory, file)
df = pd.DataFrame(data, columns=['Date', 'Power'])
df.to_csv(outname, columns=['Date', 'Power'], sep=";", index=False)
[docs] def calculate_energy(self, contracted_power):
"""
Creates a energy.csv file with the total contracted power of the community
Args:
contracted_power: total contracted power of the community
"""
print("Calculating the available energy of the community.")
df = pd.read_csv(self.path_steps_minutes + '/community.csv', sep=';')
df["Power"] = contracted_power
df["Date"] = df["Date"]
# Create csv file with the total energy consumption of the community
output_directory = os.path.join('', self.path_steps_minutes)
outname = os.path.join(output_directory, 'energy.csv')
df.to_csv(outname, columns=['Date', 'Power'], sep=";", index=False)
[docs] def execute(self, days, config_files_path, just_a_day = False, database_files = False):
"""
Executes a step of functions in order to create the consumption profiles of the dataset.
1) Gets the community information (from the JSON file) - get_community function
2) Creates the user files (with .conf extension) - create_users_files function
3) Creates the house files (with .conf extension) - create_houses_files function
4) Calculates the total contracted power of the community - calculate_contracted_power function
5) Creates the consumption profiles at 1Hz from ANTGen - create_consumption_profiles function
6) Converts the consumption profiles to 1/60Hz - convert_data_to_steps_of_one_minute function
7) Calculates community.csv with the aggregate consumption of the community - calculate_community_energy_consumption function
8) Calculates community_baseload.csv with the baseload of the community - calculate_community_baseload_consumption function
9) Calculates the community_not_baseload.csv with the non-baseload of the community - calculate_community_not_baseload_consumption function
10) Calculates energy.csv with the contracted power of the community - calculate_energy function
11) Plots the community baseline, baseload and non_baseload consumption - show_community_baseload_graph function
Args:
days: number of the days to generate data (profiles)
config_files_path: path/folder where there are the house configuration files (.conf)
just_a_day: if true, only 1 day is considered, otherwise all the days are considered
database_files: if true, converts the database files, otherwise convertes the community files (false should be used)
"""
community = self.get_community()
self.create_users_files(community)
self.create_houses_files(community)
self.contracted_power = self.calculate_contracted_power(community)
self.create_consumption_profiles(community, days, config_files_path)
self.convert_data_to_steps_of_one_minute(community, just_a_day, database_files)
self.calculate_community_energy_consumption(community)
self.calculate_community_baseload_consumption(community)
self.calculate_community_not_baseload_consumption()
self.calculate_energy(self.contracted_power)
self.show_community_baseload_graph(community, days)
# show_house_graph('0')
# show_house_graph('4')
# show_community_graph(community)