Serialization example
This example illustrates how to load a dataset from JSON, run the model on that dataset and write the output back to JSON. At the end the example is also shown for msgpack.
Load dependencies
import json
import pprint
from pathlib import Path
from pandas import DataFrame
from power_grid_model import AttributeType, ComponentAttributeFilterOptions, ComponentType, PowerGridModel
from power_grid_model.utils import json_deserialize, json_serialize
Load a dataset from a JSON file
The data is in the data/serialized-input.json file.
Load the JSON file
This is just for illustration purposes.
with Path("data/serialized_input.json").open() as fp:
data = fp.read()
pprint.pprint(json.loads(data))
{'attributes': {'node': ['id', 'u_rated'],
'source': ['id', 'node', 'status', 'u_ref', 'sk'],
'sym_load': ['id',
'node',
'status',
'type',
'p_specified',
'q_specified']},
'data': {'line': [{'c1': 4e-05,
'from_node': 1,
'from_status': 1,
'i_n': 500.0,
'id': 4,
'r1': 0.11,
'tan1': 0.1,
'to_node': 2,
'to_status': 1,
'x1': 0.12},
{'c1': 5e-05,
'from_node': 2,
'from_status': 1,
'i_n': 550.0,
'id': 5,
'r1': 0.15,
'tan1': 0.12,
'to_node': 3,
'to_status': 1,
'x1': 0.16}],
'node': [[1, 10500.0], [2, 10500.0], [3, 10500.0]],
'source': [[15, 1, 1, 1.03, 1e+20],
[16, 1, 1, 1.04, None],
{'id': 17,
'node': 1,
'rx_ratio': 0.2,
'sk': 10000000000.0,
'status': 1,
'u_ref': 1.03}],
'sym_load': [[7, 2, 1, 0, 1010000.0, 210000.0],
[8, 3, 1, 0, 1020000.0, 220000.0]]},
'is_batch': False,
'type': 'input',
'version': '1.0'}
Deserialize the JSON data
dataset = json_deserialize(data)
print("components:", list(dataset.keys()))
display(dataset[ComponentType.node])
display(DataFrame(dataset[ComponentType.node]))
components: [node, line, source, sym_load]
array([(1, 10500.), (2, 10500.), (3, 10500.)],
dtype={'names': [id, u_rated], 'formats': ['<i4', '<f8'], 'offsets': [0, 8], 'itemsize': 16, 'aligned': True})
| id | u_rated | |
|---|---|---|
| 0 | 1 | 10500.0 |
| 1 | 2 | 10500.0 |
| 2 | 3 | 10500.0 |
Run power flow calculation on the loaded input data
model = PowerGridModel(dataset)
output = model.calculate_power_flow()
display(DataFrame(output[ComponentType.node]))
| id | energized | u_pu | u | u_angle | p | q | |
|---|---|---|---|---|---|---|---|
| 0 | 1 | 1 | 1.030000 | 10815.000000 | -2.530317e-14 | 2.408998e+06 | -2.863495e+06 |
| 1 | 2 | 1 | 1.029997 | 10814.968183 | -4.398000e-03 | -1.010000e+06 | -2.100000e+05 |
| 2 | 3 | 1 | 1.029484 | 10809.581008 | -6.839956e-03 | -1.020000e+06 | -2.200000e+05 |
Serialize the output dataset
Default format
By default, the data is formatted nicely
serialized_output = json_serialize(output)
print(serialized_output)
{
"version": "1.0",
"type": "sym_output",
"is_batch": false,
"attributes": {},
"data": {
"node": [
{"id": 1, "energized": 1, "u_pu": 1.0300000000010254, "u": 10815.000000010767, "u_angle": -2.5303169101427072e-14, "p": 2408997.8394388668, "q": -2863495.3646741668},
{"id": 2, "energized": 1, "u_pu": 1.0299969698156055, "u": 10814.968183063858, "u_angle": -0.0043979998047547451, "p": -1009999.9999999704, "q": -210000.00000006545},
{"id": 3, "energized": 1, "u_pu": 1.0294839055693445, "u": 10809.581008478117, "u_angle": -0.0068399561753802384, "p": -1019999.999999999, "q": -219999.99999996895}
],
"line": [
{"id": 4, "energized": 1, "loading": 0.39953190919371073, "p_from": 2408997.8394388668, "q_from": -2863495.3646741668, "i_from": 199.76595459685538, "s_from": 3742041.7279784, "p_to": -2252625.7643675441, "q_to": 1403928.5369478234, "i_to": 141.69843328389507, "s_to": 2654305.5911384653},
{"id": 5, "energized": 1, "loading": 0.19770474338129659, "p_from": 1242625.7643675739, "q_from": -1613928.5369477719, "i_from": 108.73760885971312, "s_from": 2036880.9765532378, "p_to": -1019999.999999999, "q_to": -219999.99999996895, "i_to": 55.731992269810064, "s_to": 1043455.796859639}
],
"source": [
{"id": 15, "energized": 1, "p": -7836685.7517324276, "q": -105348495.34383295, "i": 5639.485452974508, "s": 105639571.72755387, "pf": -0.074183240461664912},
{"id": 16, "energized": 1, "p": 10248883.058391979, "q": 102488830.58119535, "i": 5498.573991718361, "s": 102999999.98954155, "pf": 0.099503719023617807},
{"id": 17, "energized": 1, "p": -0.0018078295912785419, "q": -0.010408119105616903, "i": 5.6394854529745069e-07, "s": 0.010563957172755385, "pf": -0.17113185539420431}
],
"sym_load": [
{"id": 7, "energized": 1, "p": 1010000, "q": 210000, "i": 55.071353939559145, "s": 1031600.6979447039, "pf": 0.97906098940438901},
{"id": 8, "energized": 1, "p": 1020000, "q": 220000, "i": 55.731992269810462, "s": 1043455.7968596466, "pf": 0.97752104408232876}
]
}
}
Compact serialization
In the full result, all attributes are explicitly listed for each component. In addition, all attributes are listed on a separate whiteline. This is fairly expensive memory-wise.
If you need a more memory-efficient output, you can tell the serializer to use compact lists and to avoid using redundant newlines and whitespaces.
serialized_output = json_serialize(output, use_compact_list=True, indent=-1)
print(serialized_output)
{"version":"1.0","type":"sym_output","is_batch":false,"attributes":{"node":["id","energized","u_pu","u","u_angle","p","q"],"line":["id","energized","loading","p_from","q_from","i_from","s_from","p_to","q_to","i_to","s_to"],"source":["id","energized","p","q","i","s","pf"],"sym_load":["id","energized","p","q","i","s","pf"]},"data":{"node":[[1,1,1.0300000000010254,10815.000000010767,-2.5303169101427072e-14,2408997.8394388668,-2863495.3646741668],[2,1,1.0299969698156055,10814.968183063858,-0.0043979998047547451,-1009999.9999999704,-210000.00000006545],[3,1,1.0294839055693445,10809.581008478117,-0.0068399561753802384,-1019999.999999999,-219999.99999996895]],"line":[[4,1,0.39953190919371073,2408997.8394388668,-2863495.3646741668,199.76595459685538,3742041.7279784,-2252625.7643675441,1403928.5369478234,141.69843328389507,2654305.5911384653],[5,1,0.19770474338129659,1242625.7643675739,-1613928.5369477719,108.73760885971312,2036880.9765532378,-1019999.999999999,-219999.99999996895,55.731992269810064,1043455.796859639]],"source":[[15,1,-7836685.7517324276,-105348495.34383295,5639.485452974508,105639571.72755387,-0.074183240461664912],[16,1,10248883.058391979,102488830.58119535,5498.573991718361,102999999.98954155,0.099503719023617807],[17,1,-0.0018078295912785419,-0.010408119105616903,5.6394854529745069e-07,0.010563957172755385,-0.17113185539420431]],"sym_load":[[7,1,1010000,210000,55.071353939559145,1031600.6979447039,0.97906098940438901],[8,1,1020000,220000,55.731992269810462,1043455.7968596466,0.97752104408232876]]}}
The compact result is still valid JSON
pprint.pprint(json.loads(serialized_output))
{'attributes': {'line': ['id',
'energized',
'loading',
'p_from',
'q_from',
'i_from',
's_from',
'p_to',
'q_to',
'i_to',
's_to'],
'node': ['id', 'energized', 'u_pu', 'u', 'u_angle', 'p', 'q'],
'source': ['id', 'energized', 'p', 'q', 'i', 's', 'pf'],
'sym_load': ['id', 'energized', 'p', 'q', 'i', 's', 'pf']},
'data': {'line': [[4,
1,
0.39953190919371073,
2408997.839438867,
-2863495.364674167,
199.76595459685538,
3742041.7279784,
-2252625.764367544,
1403928.5369478234,
141.69843328389507,
2654305.5911384653],
[5,
1,
0.1977047433812966,
1242625.764367574,
-1613928.536947772,
108.73760885971312,
2036880.9765532378,
-1019999.999999999,
-219999.99999996895,
55.731992269810064,
1043455.796859639]],
'node': [[1,
1,
1.0300000000010254,
10815.000000010767,
-2.5303169101427072e-14,
2408997.839438867,
-2863495.364674167],
[2,
1,
1.0299969698156055,
10814.968183063858,
-0.004397999804754745,
-1009999.9999999704,
-210000.00000006545],
[3,
1,
1.0294839055693445,
10809.581008478117,
-0.0068399561753802384,
-1019999.999999999,
-219999.99999996895]],
'source': [[15,
1,
-7836685.751732428,
-105348495.34383295,
5639.485452974508,
105639571.72755387,
-0.07418324046166491],
[16,
1,
10248883.058391979,
102488830.58119535,
5498.573991718361,
102999999.98954155,
0.0995037190236178],
[17,
1,
-0.001807829591278542,
-0.010408119105616903,
5.639485452974507e-07,
0.010563957172755385,
-0.1711318553942043]],
'sym_load': [[7,
1,
1010000,
210000,
55.071353939559145,
1031600.6979447039,
0.979060989404389],
[8,
1,
1020000,
220000,
55.73199226981046,
1043455.7968596466,
0.9775210440823288]]},
'is_batch': False,
'type': 'sym_output',
'version': '1.0'}
Msgpack serialization
To have even higher performance and smaller space, you can use the binary format msgpack. The example below shows a round trip to dump and load msgpack data, and intantiate model.
Serialize to msgpack
We can serialize the output data into msgpack binary with and without compact list. The result is a bytes object. See the resulted differences in length of the data. The differences will be significant when you have a large dataset.
from power_grid_model.utils import msgpack_serialize
msgpack_data_not_compact = msgpack_serialize(output, use_compact_list=False)
msgpack_data_compact = msgpack_serialize(output, use_compact_list=True)
print(f"Type of the returned objects: {type(msgpack_data_not_compact)}, {type(msgpack_data_compact)}")
print(f"Length of not-compact data: {len(msgpack_data_not_compact)}")
print(f"Length of compact data: {len(msgpack_data_compact)}")
Type of the returned objects: <class 'bytes'>, <class 'bytes'>
Length of not-compact data: 993
Length of compact data: 818
Deserialize from msgpack
We can deserialize the data we just created. We then print the node result. Note that they are exactly the same.
from power_grid_model.utils import msgpack_deserialize
output_data_not_compact = msgpack_deserialize(msgpack_data_not_compact)
output_data_compact = msgpack_deserialize(msgpack_data_compact)
print("----Node result from not compact data----")
print(DataFrame(output_data_not_compact[ComponentType.node]))
print("----Node result from compact data----")
print(DataFrame(output_data_compact[ComponentType.node]))
----Node result from not compact data----
id energized u_pu u u_angle p \
0 1 1 1.030000 10815.000000 -2.530317e-14 2.408998e+06
1 2 1 1.029997 10814.968183 -4.398000e-03 -1.010000e+06
2 3 1 1.029484 10809.581008 -6.839956e-03 -1.020000e+06
q
0 -2.863495e+06
1 -2.100000e+05
2 -2.200000e+05
----Node result from compact data----
id energized u_pu u u_angle p \
0 1 1 1.030000 10815.000000 -2.530317e-14 2.408998e+06
1 2 1 1.029997 10814.968183 -4.398000e-03 -1.010000e+06
2 3 1 1.029484 10809.581008 -6.839956e-03 -1.020000e+06
q
0 -2.863495e+06
1 -2.100000e+05
2 -2.200000e+05
Selective deserialization and dataset format
To control the dataset returned by the deserialization functionality, you can use the data_filter argument.
Deserialization to columnar dataset
dataset = json_deserialize(data, data_filter=ComponentAttributeFilterOptions.everything)
print("components:", list(dataset.keys()))
display(dataset[ComponentType.node])
components: [node, line, source, sym_load]
{id: array([1, 2, 3], dtype=int32), u_rated: array([10500., 10500., 10500.])}
Deserialized data format selection per component type
To select specific components and data formats for the deserialized data, provide a dictionary of components and their desired output types to the data_filter.
dataset = json_deserialize(
data,
data_filter={
ComponentType.node: None, # nodes in a row-based data format
ComponentType.source: [
AttributeType.id,
AttributeType.node,
AttributeType.status,
AttributeType.u_ref,
AttributeType.sk,
], # only specific attributes
ComponentType.sym_load: ComponentAttributeFilterOptions.everything, # all attributes as columns
ComponentType.line: ComponentAttributeFilterOptions.relevant, # only attributes that are not null/nan
},
)
print("components:", list(dataset.keys()))
print("node attributes:", list(dataset[ComponentType.node].dtype.names))
print("source attributes:", list(dataset[ComponentType.source].keys()))
print("sym_load attributes:", list(dataset[ComponentType.sym_load].keys()))
print("line attributes:", list(dataset[ComponentType.line].keys()))
components: [node, line, source, sym_load]
node attributes: [id, u_rated]
source attributes: [id, node, status, u_ref, sk]
sym_load attributes: [id, node, status, type, p_specified, q_specified]
line attributes: [id, from_node, to_node, from_status, to_status, r1, x1, c1, tan1, i_n]
A columnar dataset can also be serialized again, as one would expect.
serialized_output = json_serialize(output)
print(serialized_output)
{
"version": "1.0",
"type": "sym_output",
"is_batch": false,
"attributes": {},
"data": {
"node": [
{"id": 1, "energized": 1, "u_pu": 1.0300000000010254, "u": 10815.000000010767, "u_angle": -2.5303169101427072e-14, "p": 2408997.8394388668, "q": -2863495.3646741668},
{"id": 2, "energized": 1, "u_pu": 1.0299969698156055, "u": 10814.968183063858, "u_angle": -0.0043979998047547451, "p": -1009999.9999999704, "q": -210000.00000006545},
{"id": 3, "energized": 1, "u_pu": 1.0294839055693445, "u": 10809.581008478117, "u_angle": -0.0068399561753802384, "p": -1019999.999999999, "q": -219999.99999996895}
],
"line": [
{"id": 4, "energized": 1, "loading": 0.39953190919371073, "p_from": 2408997.8394388668, "q_from": -2863495.3646741668, "i_from": 199.76595459685538, "s_from": 3742041.7279784, "p_to": -2252625.7643675441, "q_to": 1403928.5369478234, "i_to": 141.69843328389507, "s_to": 2654305.5911384653},
{"id": 5, "energized": 1, "loading": 0.19770474338129659, "p_from": 1242625.7643675739, "q_from": -1613928.5369477719, "i_from": 108.73760885971312, "s_from": 2036880.9765532378, "p_to": -1019999.999999999, "q_to": -219999.99999996895, "i_to": 55.731992269810064, "s_to": 1043455.796859639}
],
"source": [
{"id": 15, "energized": 1, "p": -7836685.7517324276, "q": -105348495.34383295, "i": 5639.485452974508, "s": 105639571.72755387, "pf": -0.074183240461664912},
{"id": 16, "energized": 1, "p": 10248883.058391979, "q": 102488830.58119535, "i": 5498.573991718361, "s": 102999999.98954155, "pf": 0.099503719023617807},
{"id": 17, "energized": 1, "p": -0.0018078295912785419, "q": -0.010408119105616903, "i": 5.6394854529745069e-07, "s": 0.010563957172755385, "pf": -0.17113185539420431}
],
"sym_load": [
{"id": 7, "energized": 1, "p": 1010000, "q": 210000, "i": 55.071353939559145, "s": 1031600.6979447039, "pf": 0.97906098940438901},
{"id": 8, "energized": 1, "p": 1020000, "q": 220000, "i": 55.731992269810462, "s": 1043455.7968596466, "pf": 0.97752104408232876}
]
}
}