Serialization example
This example illustrates how to load a dataset from JSON, run the model on that dataset and write the output back to JSON. At the end the example is also shown for msgpack.
Load dependencies
import json
import pprint
import warnings
with warnings.catch_warnings(action="ignore", category=DeprecationWarning):
# suppress warning about pyarrow as future required dependency
from pandas import DataFrame
from power_grid_model import PowerGridModel
from power_grid_model.utils import json_deserialize, json_serialize
Load a dataset from a JSON file
The data is in the data/serialized-input.json file.
Load the JSON file
with open("data/serialized_input.json") as fp:
data = fp.read()
pprint.pprint(json.loads(data))
{'attributes': {'node': ['id', 'u_rated'],
'source': ['id', 'node', 'status', 'u_ref', 'sk'],
'sym_load': ['id',
'node',
'status',
'type',
'p_specified',
'q_specified']},
'data': {'line': [{'c1': 4e-05,
'from_node': 1,
'from_status': 1,
'i_n': 500.0,
'id': 4,
'r1': 0.11,
'tan1': 0.1,
'to_node': 2,
'to_status': 1,
'x1': 0.12},
{'c1': 5e-05,
'from_node': 2,
'from_status': 1,
'i_n': 550.0,
'id': 5,
'r1': 0.15,
'tan1': 0.12,
'to_node': 3,
'to_status': 1,
'x1': 0.16}],
'node': [[1, 10500.0], [2, 10500.0], [3, 10500.0]],
'source': [[15, 1, 1, 1.03, 1e+20],
[16, 1, 1, 1.04, None],
{'id': 17,
'node': 1,
'rx_ratio': 0.2,
'sk': 10000000000.0,
'status': 1,
'u_ref': 1.03}],
'sym_load': [[7, 2, 1, 0, 1010000.0, 210000.0],
[8, 3, 1, 0, 1020000.0, 220000.0]]},
'is_batch': False,
'type': 'input',
'version': '1.0'}
Deserialize the JSON data
dataset = json_deserialize(data)
print("components:", dataset.keys())
display(DataFrame(dataset["node"]))
components: dict_keys(['node', 'line', 'source', 'sym_load'])
| id | u_rated | |
|---|---|---|
| 0 | 1 | 10500.0 |
| 1 | 2 | 10500.0 |
| 2 | 3 | 10500.0 |
Run power flow calculation on the loaded input data
model = PowerGridModel(dataset)
output = model.calculate_power_flow()
display(DataFrame(output["node"]))
| id | energized | u_pu | u | u_angle | p | q | |
|---|---|---|---|---|---|---|---|
| 0 | 1 | 1 | 1.030000 | 10815.000000 | -2.527860e-14 | 2.408998e+06 | -2.863495e+06 |
| 1 | 2 | 1 | 1.029997 | 10814.968183 | -4.398000e-03 | -1.010000e+06 | -2.100000e+05 |
| 2 | 3 | 1 | 1.029484 | 10809.581008 | -6.839956e-03 | -1.020000e+06 | -2.200000e+05 |
Serialize the output dataset
Default format
By default, the data is formatted nicely
serialized_output = json_serialize(output)
print(serialized_output)
{
"version": "1.0",
"type": "sym_output",
"is_batch": false,
"attributes": {},
"data": {
"node": [
{"id": 1, "energized": 1, "u_pu": 1.030000000001025, "u": 10815.00000001077, "u_angle": -2.527859536004767e-14, "p": 2408997.839438984, "q": -2863495.364673933},
{"id": 2, "energized": 1, "u_pu": 1.029996969815605, "u": 10814.96818306385, "u_angle": -0.004397999804754551, "p": -1010000.000000146, "q": -210000.0000001819},
{"id": 3, "energized": 1, "u_pu": 1.029483905569344, "u": 10809.58100847811, "u_angle": -0.006839956175380042, "p": -1019999.99999994, "q": -220000.000000028}
],
"line": [
{"id": 4, "energized": 1, "loading": 0.3995319091936997, "p_from": 2408997.839438984, "q_from": -2863495.364673933, "i_from": 199.7659545968498, "s_from": 3742041.727978296, "p_to": -2252625.764367662, "q_to": 1403928.536947472, "i_to": 141.6984332838906, "s_to": 2654305.591138379},
{"id": 5, "energized": 1, "loading": 0.1977047433812886, "p_from": 1242625.764367516, "q_from": -1613928.536947712, "i_from": 108.7376088597087, "s_from": 2036880.976553155, "p_to": -1019999.99999994, "q_to": -220000.000000028, "i_to": 55.73199226980768, "s_to": 1043455.796859594}
],
"source": [
{"id": 15, "energized": 1, "p": -7840728.150705829, "q": -105359100.0736002, "i": 5640.066029671385, "s": 105650447.1654151, "pf": -0.07421386620758674},
{"id": 16, "energized": 1, "p": 10248883.05839162, "q": 102488830.5811948, "i": 5498.573991718331, "s": 102999999.989541, "pf": 0.09950371902361486},
{"id": 17, "energized": 1, "p": -0.001808335374213852, "q": -0.01040913507631103, "i": 5.640066029671386e-07, "s": 0.01056504471654151, "pf": -0.1711621126773437}
],
"sym_load": [
{"id": 7, "energized": 1, "p": 1010000, "q": 210000, "i": 55.07135393955917, "s": 1031600.697944704, "pf": 0.979060989404389},
{"id": 8, "energized": 1, "p": 1020000, "q": 220000, "i": 55.7319922698105, "s": 1043455.796859647, "pf": 0.9775210440823288}
]
}
}
Compact serialization
In the full result, all attributes are explicitly listed for each component. In addition, all attributes are listed on a separate whiteline. This is fairly expensive memory-wise.
If you need a more memory-efficient output, you can tell the serializer to use compact lists and to avoid using redundant newlines and whitespaces.
serialized_output = json_serialize(output, use_compact_list=True, indent=-1)
print(serialized_output)
{"version":"1.0","type":"sym_output","is_batch":false,"attributes":{"node":["id","energized","u_pu","u","u_angle","p","q"],"line":["id","energized","loading","p_from","q_from","i_from","s_from","p_to","q_to","i_to","s_to"],"source":["id","energized","p","q","i","s","pf"],"sym_load":["id","energized","p","q","i","s","pf"]},"data":{"node":[[1,1,1.030000000001025,10815.00000001077,-2.527859536004767e-14,2408997.839438984,-2863495.364673933],[2,1,1.029996969815605,10814.96818306385,-0.004397999804754551,-1010000.000000146,-210000.0000001819],[3,1,1.029483905569344,10809.58100847811,-0.006839956175380042,-1019999.99999994,-220000.000000028]],"line":[[4,1,0.3995319091936997,2408997.839438984,-2863495.364673933,199.7659545968498,3742041.727978296,-2252625.764367662,1403928.536947472,141.6984332838906,2654305.591138379],[5,1,0.1977047433812886,1242625.764367516,-1613928.536947712,108.7376088597087,2036880.976553155,-1019999.99999994,-220000.000000028,55.73199226980768,1043455.796859594]],"source":[[15,1,-7840728.150705829,-105359100.0736002,5640.066029671385,105650447.1654151,-0.07421386620758674],[16,1,10248883.05839162,102488830.5811948,5498.573991718331,102999999.989541,0.09950371902361486],[17,1,-0.001808335374213852,-0.01040913507631103,5.640066029671386e-07,0.01056504471654151,-0.1711621126773437]],"sym_load":[[7,1,1010000,210000,55.07135393955917,1031600.697944704,0.979060989404389],[8,1,1020000,220000,55.7319922698105,1043455.796859647,0.9775210440823288]]}}
The compact result is still valid JSON
pprint.pprint(json.loads(serialized_output))
{'attributes': {'line': ['id',
'energized',
'loading',
'p_from',
'q_from',
'i_from',
's_from',
'p_to',
'q_to',
'i_to',
's_to'],
'node': ['id', 'energized', 'u_pu', 'u', 'u_angle', 'p', 'q'],
'source': ['id', 'energized', 'p', 'q', 'i', 's', 'pf'],
'sym_load': ['id', 'energized', 'p', 'q', 'i', 's', 'pf']},
'data': {'line': [[4,
1,
0.3995319091936997,
2408997.839438984,
-2863495.364673933,
199.7659545968498,
3742041.727978296,
-2252625.764367662,
1403928.536947472,
141.6984332838906,
2654305.591138379],
[5,
1,
0.1977047433812886,
1242625.764367516,
-1613928.536947712,
108.7376088597087,
2036880.976553155,
-1019999.99999994,
-220000.000000028,
55.73199226980768,
1043455.796859594]],
'node': [[1,
1,
1.030000000001025,
10815.00000001077,
-2.527859536004767e-14,
2408997.839438984,
-2863495.364673933],
[2,
1,
1.029996969815605,
10814.96818306385,
-0.004397999804754551,
-1010000.000000146,
-210000.0000001819],
[3,
1,
1.029483905569344,
10809.58100847811,
-0.006839956175380042,
-1019999.99999994,
-220000.000000028]],
'source': [[15,
1,
-7840728.150705829,
-105359100.0736002,
5640.066029671385,
105650447.1654151,
-0.07421386620758674],
[16,
1,
10248883.05839162,
102488830.5811948,
5498.573991718331,
102999999.989541,
0.09950371902361486],
[17,
1,
-0.001808335374213852,
-0.01040913507631103,
5.640066029671386e-07,
0.01056504471654151,
-0.1711621126773437]],
'sym_load': [[7,
1,
1010000,
210000,
55.07135393955917,
1031600.697944704,
0.979060989404389],
[8,
1,
1020000,
220000,
55.7319922698105,
1043455.796859647,
0.9775210440823288]]},
'is_batch': False,
'type': 'sym_output',
'version': '1.0'}
Msgpack serialization
To have even higher performance and smaller space, you can use the binary format msgpack. The example below shows a round trip to dump and load msgpack data, and intantiate model.
Serialize to msgpack
We can serialize the output data into msgpack binary with and without compact list. The result is a bytes object. See the resulted differences in length of the data. The differences will be significant when you have a large dataset.
from power_grid_model.utils import msgpack_serialize
msgpack_data_not_compact = msgpack_serialize(output, use_compact_list=False)
msgpack_data_compact = msgpack_serialize(output, use_compact_list=True)
print(f"Type of the returned objects: {type(msgpack_data_not_compact)}, {type(msgpack_data_compact)}")
print(f"Length of not-compact data: {len(msgpack_data_not_compact)}")
print(f"Length of compact data: {len(msgpack_data_compact)}")
Type of the returned objects: <class 'bytes'>, <class 'bytes'>
Length of not-compact data: 977
Length of compact data: 802
Deserialize from msgpack
We can deserialize the data we just created. We then print the node result. Note that they are exactly the same.
from power_grid_model.utils import msgpack_deserialize
output_data_not_compact = msgpack_deserialize(msgpack_data_not_compact)
output_data_compact = msgpack_deserialize(msgpack_data_compact)
print("----Node result from not compact data----")
print(DataFrame(output_data_not_compact["node"]))
print("----Node result from compact data----")
print(DataFrame(output_data_compact["node"]))
----Node result from not compact data----
id energized u_pu u u_angle p \
0 1 1 1.030000 10815.000000 -2.527860e-14 2.408998e+06
1 2 1 1.029997 10814.968183 -4.398000e-03 -1.010000e+06
2 3 1 1.029484 10809.581008 -6.839956e-03 -1.020000e+06
q
0 -2.863495e+06
1 -2.100000e+05
2 -2.200000e+05
----Node result from compact data----
id energized u_pu u u_angle p \
0 1 1 1.030000 10815.000000 -2.527860e-14 2.408998e+06
1 2 1 1.029997 10814.968183 -4.398000e-03 -1.010000e+06
2 3 1 1.029484 10809.581008 -6.839956e-03 -1.020000e+06
q
0 -2.863495e+06
1 -2.100000e+05
2 -2.200000e+05