From ebe1ccc0108d14ed5b561f22dc9f401d3ffab1b5 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Tue, 29 Dec 2020 00:25:17 +0100 Subject: [PATCH 01/26] started work on generating data for FCR script, finished config file validation, added example config file --- FCRgendata.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ config.json | 15 +++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 FCRgendata.py create mode 100644 config.json diff --git a/FCRgendata.py b/FCRgendata.py new file mode 100644 index 0000000..5f1e01f --- /dev/null +++ b/FCRgendata.py @@ -0,0 +1,59 @@ +from pathlib import Path +import xml.etree.ElementTree as ET +import sys +import logging +import json + + +logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) + +CONFIG_PATH: Path = None + +if len(sys.argv) != 2: + logging.critical(f"received {len(sys.argv) - 1} arguments, required one: config file path.") + sys.exit(1) +else: + CONFIG_PATH = Path(sys.argv[1]) + + +def validate_config(conf_p: Path): + ''' validates given config path, config path structure and it content ''' + + if not conf_p.is_file(): + logging.critical(f"given path: {conf_p} doesn't lead to file") + sys.exit(1) + + j_conf = None + with conf_p.open() as f: + try: + j_conf = json.load(f) + except json.decoder.JSONDecodeError as err: + logging.critical(f"error decoding json config") + sys.exit(1) + + for label in ('request', 'AvgResponseTimeTableFilePath'): + if label not in j_conf: + logging.critical(f'given config json is missing "request" label') + sys.exit(1) + + for label in ('camelModelFilePath', 'cpProblemFilePath', 'nodeCandidatesFilePath'): + if label not in j_conf['request']: + logging.critical(f'given config json is missing "{label}" label in "request" object label') + sys.exit(1) + + if not Path(j_conf['request'][label]).is_file(): + logging.critical(f"given path: {label}:{j_conf['request'][label]} in config doesn't lead to file") + sys.exit(1) + + if not Path(j_conf['AvgResponseTimeTableFilePath']).is_file(): + logging.critical(f'given path to "AvgResponseTimeTableFilePath" is invalid') + sys.exit(1) + + if 'outpath' in j_conf and not Path(j_conf['outpath']).parent.is_dir(): + logging.critical(f'in given outpath file can\'t be created') + sys.exit(1) + + return j_conf + + +validate_config(CONFIG_PATH) \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..7fbfc5e --- /dev/null +++ b/config.json @@ -0,0 +1,15 @@ +{ + "request": { + "applicationId": "FCRwithDLMSApp", + "camelModelFilePath" : "/home/szysad/mimuw/3rok/ZPP/FCR-problems/podobno-dane-co-dzialaja/FCRnewWithSensor2CoresOneInstance.xmi", + "cpProblemFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/podobno-dane-co-dzialaja/FCR-CP.xmi", + "nodeCandidatesFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/podobno-dane-co-dzialaja/FCR-NodeCandidates", + "watermark": { + "user": "mrozanska", + "system": "UI", + "date": "2017-11-23T16: 41: 41+0000", + "uuid": "fb6280ec-1ab8-11e7-93ae-92361f002671" + } + }, + "AvgResponseTimeTableFilePath": "/home/szysad/mimuw/3rok/ZPP/time-series-data/time-series-data/secure-document​/deployment-reconfiguration-range-2-to-2/2020-10-30 to 2020-10-30/V 1.0 - raw data/AvgResponseTimeTable.csv" +} \ No newline at end of file -- GitLab From a7a199a39726af1c22e5170b6048258bc0310dc5 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Tue, 29 Dec 2020 12:12:10 +0100 Subject: [PATCH 02/26] refractor, changed json validation using jsonschema, added AvgResponseTimeTable reading --- FCRgendata.py | 59 ---- FCRgendata/Pipfile | 15 + FCRgendata/Pipfile.lock | 381 ++++++++++++++++++++++++++ config.json => FCRgendata/config.json | 0 FCRgendata/setup.py | 222 +++++++++++++++ FCRgendata/src/__init__.py | 0 FCRgendata/src/main.py | 30 ++ FCRgendata/src/validate_config.py | 79 ++++++ 8 files changed, 727 insertions(+), 59 deletions(-) delete mode 100644 FCRgendata.py create mode 100644 FCRgendata/Pipfile create mode 100644 FCRgendata/Pipfile.lock rename config.json => FCRgendata/config.json (100%) create mode 100644 FCRgendata/setup.py create mode 100644 FCRgendata/src/__init__.py create mode 100644 FCRgendata/src/main.py create mode 100644 FCRgendata/src/validate_config.py diff --git a/FCRgendata.py b/FCRgendata.py deleted file mode 100644 index 5f1e01f..0000000 --- a/FCRgendata.py +++ /dev/null @@ -1,59 +0,0 @@ -from pathlib import Path -import xml.etree.ElementTree as ET -import sys -import logging -import json - - -logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) - -CONFIG_PATH: Path = None - -if len(sys.argv) != 2: - logging.critical(f"received {len(sys.argv) - 1} arguments, required one: config file path.") - sys.exit(1) -else: - CONFIG_PATH = Path(sys.argv[1]) - - -def validate_config(conf_p: Path): - ''' validates given config path, config path structure and it content ''' - - if not conf_p.is_file(): - logging.critical(f"given path: {conf_p} doesn't lead to file") - sys.exit(1) - - j_conf = None - with conf_p.open() as f: - try: - j_conf = json.load(f) - except json.decoder.JSONDecodeError as err: - logging.critical(f"error decoding json config") - sys.exit(1) - - for label in ('request', 'AvgResponseTimeTableFilePath'): - if label not in j_conf: - logging.critical(f'given config json is missing "request" label') - sys.exit(1) - - for label in ('camelModelFilePath', 'cpProblemFilePath', 'nodeCandidatesFilePath'): - if label not in j_conf['request']: - logging.critical(f'given config json is missing "{label}" label in "request" object label') - sys.exit(1) - - if not Path(j_conf['request'][label]).is_file(): - logging.critical(f"given path: {label}:{j_conf['request'][label]} in config doesn't lead to file") - sys.exit(1) - - if not Path(j_conf['AvgResponseTimeTableFilePath']).is_file(): - logging.critical(f'given path to "AvgResponseTimeTableFilePath" is invalid') - sys.exit(1) - - if 'outpath' in j_conf and not Path(j_conf['outpath']).parent.is_dir(): - logging.critical(f'in given outpath file can\'t be created') - sys.exit(1) - - return j_conf - - -validate_config(CONFIG_PATH) \ No newline at end of file diff --git a/FCRgendata/Pipfile b/FCRgendata/Pipfile new file mode 100644 index 0000000..1b13b8c --- /dev/null +++ b/FCRgendata/Pipfile @@ -0,0 +1,15 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +jsonschema = "*" +setuptools = "*" +pipenv-setup = "*" +numpy = "*" + +[requires] +python_version = "3.8" diff --git a/FCRgendata/Pipfile.lock b/FCRgendata/Pipfile.lock new file mode 100644 index 0000000..2770d65 --- /dev/null +++ b/FCRgendata/Pipfile.lock @@ -0,0 +1,381 @@ +{ + "_meta": { + "hash": { + "sha256": "555331b6680f90953e06c955358095d7cf2a7f4f74b3fbd998491149e5ea7a16" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.3.0" + }, + "black": { + "hashes": [ + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" + ], + "markers": "python_version >= '3.6'", + "version": "==19.10b0" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "cerberus": { + "hashes": [ + "sha256:302e6694f206dd85cb63f13fd5025b31ab6d38c99c50c6d769f8fa0b0f299589" + ], + "version": "==1.3.2" + }, + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.4" + }, + "distlib": { + "hashes": [ + "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", + "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" + ], + "version": "==0.3.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "jsonschema": { + "hashes": [ + "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", + "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" + ], + "index": "pypi", + "version": "==3.2.0" + }, + "numpy": { + "hashes": [ + "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db", + "sha256:09c12096d843b90eafd01ea1b3307e78ddd47a55855ad402b157b6c4862197ce", + "sha256:13d166f77d6dc02c0a73c1101dd87fdf01339febec1030bd810dcd53fff3b0f1", + "sha256:141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512", + "sha256:16c1b388cc31a9baa06d91a19366fb99ddbe1c7b205293ed072211ee5bac1ed2", + "sha256:18bed2bcb39e3f758296584337966e68d2d5ba6aab7e038688ad53c8f889f757", + "sha256:1aeef46a13e51931c0b1cf8ae1168b4a55ecd282e6688fdb0a948cc5a1d5afb9", + "sha256:27d3f3b9e3406579a8af3a9f262f5339005dd25e0ecf3cf1559ff8a49ed5cbf2", + "sha256:2a2740aa9733d2e5b2dfb33639d98a64c3b0f24765fed86b0fd2aec07f6a0a08", + "sha256:4377e10b874e653fe96985c05feed2225c912e328c8a26541f7fc600fb9c637b", + "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb", + "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc", + "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac", + "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83", + "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36", + "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387", + "sha256:8cac8790a6b1ddf88640a9267ee67b1aee7a57dfa2d2dd33999d080bc8ee3a0f", + "sha256:8ece138c3a16db8c1ad38f52eb32be6086cc72f403150a79336eb2045723a1ad", + "sha256:9eeb7d1d04b117ac0d38719915ae169aa6b61fca227b0b7d198d43728f0c879c", + "sha256:a09f98011236a419ee3f49cedc9ef27d7a1651df07810ae430a6b06576e0b414", + "sha256:a5d897c14513590a85774180be713f692df6fa8ecf6483e561a6d47309566f37", + "sha256:ad6f2ff5b1989a4899bf89800a671d71b1612e5ff40866d1f4d8bcf48d4e5764", + "sha256:c42c4b73121caf0ed6cd795512c9c09c52a7287b04d105d112068c1736d7c753", + "sha256:cb1017eec5257e9ac6209ac172058c430e834d5d2bc21961dceeb79d111e5909", + "sha256:d6c7bb82883680e168b55b49c70af29b84b84abb161cbac2800e8fcb6f2109b6", + "sha256:e452dc66e08a4ce642a961f134814258a082832c78c90351b75c41ad16f79f63", + "sha256:e5b6ed0f0b42317050c88022349d994fe72bfe35f5908617512cd8c8ef9da2a9", + "sha256:e9b30d4bd69498fc0c3fe9db5f62fffbb06b8eb9321f92cc970f2969be5e3949", + "sha256:ec149b90019852266fec2341ce1db513b843e496d5a8e8cdb5ced1923a92faab", + "sha256:edb01671b3caae1ca00881686003d16c2209e07b7ef8b7639f1867852b948f7c", + "sha256:f0d3929fe88ee1c155129ecd82f981b8856c5d97bcb0d5f23e9b4242e79d1de3", + "sha256:f29454410db6ef8126c83bd3c968d143304633d45dc57b51252afbd79d700893", + "sha256:fe45becb4c2f72a0907c1d0246ea6449fe7a9e2293bb0e11c4e9a32bb0930a15", + "sha256:fedbd128668ead37f33917820b704784aff695e0019309ad446a6d0b065b57e4" + ], + "index": "pypi", + "version": "==1.19.4" + }, + "orderedmultidict": { + "hashes": [ + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" + ], + "version": "==1.0.1" + }, + "packaging": { + "hashes": [ + "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858", + "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.8" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, + "pep517": { + "hashes": [ + "sha256:3985b91ebf576883efe5fa501f42a16de2607684f3797ddba7202b71b7d0da51", + "sha256:aeb78601f2d1aa461960b43add204cc7955667687fbcf9cdb5170f00556f117f" + ], + "version": "==0.9.1" + }, + "pip-shims": { + "hashes": [ + "sha256:05b00ade9d1e686a98bb656dd9b0608a933897283dc21913fad6ea5409ff7e91", + "sha256:16ca9f87485667b16b978b68a1aae4f9cc082c0fa018aed28567f9f34a590569" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.5.3" + }, + "pipenv-setup": { + "hashes": [ + "sha256:8a439aff7b16e18d7e07702c9186fc5fe86156679eace90e10c2578a43bd7af1", + "sha256:e1bfd55c1152024e762f1c17f6189fcb073166509e7c0228870f7ea160355648" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pipfile": { + "hashes": [ + "sha256:f7d9f15de8b660986557eb3cc5391aa1a16207ac41bc378d03f414762d36c984" + ], + "version": "==0.0.2" + }, + "plette": { + "extras": [ + "validation" + ], + "hashes": [ + "sha256:46402c03e36d6eadddad2a5125990e322dd74f98160c8f2dcd832b2291858a26", + "sha256:d6c9b96981b347bddd333910b753b6091a2c1eb2ef85bb373b4a67c9d91dca16" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.3" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "pyrsistent": { + "hashes": [ + "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" + ], + "markers": "python_version >= '3.5'", + "version": "==0.17.3" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.1" + }, + "regex": { + "hashes": [ + "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", + "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", + "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", + "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", + "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", + "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", + "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", + "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", + "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", + "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", + "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", + "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", + "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", + "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", + "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", + "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", + "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", + "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", + "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", + "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", + "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", + "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", + "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", + "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", + "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", + "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", + "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", + "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", + "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", + "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", + "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", + "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", + "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", + "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", + "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", + "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", + "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", + "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", + "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", + "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", + "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + ], + "version": "==2020.11.13" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.25.1" + }, + "requirementslib": { + "hashes": [ + "sha256:50d20f27e4515a2393695b0d886219598302163438ae054253147b2bad9b4a44", + "sha256:9c1e8666ca4512724cdd1739adcc7df19ec7ad2ed21f0e748f9631ad6b54f321" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.5.16" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "tomlkit": { + "hashes": [ + "sha256:6babbd33b17d5c9691896b0e68159215a9387ebfa938aa3ac42f4a4beeb2b831", + "sha256:ac57f29693fab3e309ea789252fcce3061e19110085aa31af5446ca749325618" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.7.0" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072", + "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298", + "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "version": "==1.4.1" + }, + "urllib3": { + "hashes": [ + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.2" + }, + "vistir": { + "hashes": [ + "sha256:a37079cdbd85d31a41cdd18457fe521e15ec08b255811e81aa061fd5f48a20fb", + "sha256:eff1d19ef50c703a329ed294e5ec0b0fbb35b96c1b3ee6dcdb266dddbe1e935a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.2" + }, + "wheel": { + "hashes": [ + "sha256:78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", + "sha256:e11eefd162658ea59a60a0f6c7d493a7190ea4b9a85e335b33489d9f17e0245e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.36.2" + } + }, + "develop": {} +} diff --git a/config.json b/FCRgendata/config.json similarity index 100% rename from config.json rename to FCRgendata/config.json diff --git a/FCRgendata/setup.py b/FCRgendata/setup.py new file mode 100644 index 0000000..ad61eca --- /dev/null +++ b/FCRgendata/setup.py @@ -0,0 +1,222 @@ +"""A setuptools based setup module. +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +Modified by Madoshakalaka@Github (dependency links added) +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +from os import path + +# io.open is needed for projects that support Python 2.7 +# It ensures open() defaults to text mode with universal newlines, +# and accepts an argument to specify the text encoding +# Python 3 only projects can skip this import +from io import open + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, "README.md"), encoding="utf-8") as f: + long_description = f.read() + +# Arguments marked as "Required" below must be included for upload to PyPI. +# Fields marked as "Optional" may be commented out. + +setup( + # This is the name of your project. The first time you publish this + # package, this name will be registered for you. It will determine how + # users can install this project, e.g.: + # + # $ pip install sampleproject + # + # And where it will live on PyPI: https://pypi.org/project/sampleproject/ + # + # There are some restrictions on what makes a valid project name + # specification here: + # https://packaging.python.org/specifications/core-metadata/#name + name="FCRgenData", # Required + # Versions should comply with PEP 440: + # https://www.python.org/dev/peps/pep-0440/ + # + # For a discussion on single-sourcing the version across setup.py and the + # project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version="0.0.0", # Required + # This is a one-line description or tagline of what your project does. This + # corresponds to the "Summary" metadata field: + # https://packaging.python.org/specifications/core-metadata/#summary + #description="A sample Python project", # Optional + # This is an optional longer description of your project that represents + # the body of text which users will see when they visit PyPI. + # + # Often, this is the same as your README, so you can just read it in from + # that file directly (as we have already done above) + # + # This field corresponds to the "Description" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-optional + #long_description=long_description, # Optional + # Denotes that our long_description is in Markdown; valid values are + # text/plain, text/x-rst, and text/markdown + # + # Optional if long_description is written in reStructuredText (rst) but + # required for plain-text or Markdown; if unspecified, "applications should + # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and + # fall back to text/plain if it is not valid rst" (see link below) + # + # This field corresponds to the "Description-Content-Type" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional + #long_description_content_type="text/markdown", # Optional (see note above) + # This should be a valid link to your project's main homepage. + # + # This field corresponds to the "Home-Page" metadata field: + # https://packaging.python.org/specifications/core-metadata/#home-page-optional + #url="https://github.com/pypa/sampleproject", # Optional + # This should be your name or the name of the organization which owns the + # project. + #author="The Python Packaging Authority", # Optional + # This should be a valid email address corresponding to the author listed + # above. + #author_email="pypa-dev@googlegroups.com", # Optional + # Classifiers help users find your project by categorizing it. + # + # For a list of valid classifiers, see https://pypi.org/classifiers/ + classifiers=[ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 3 - Alpha", + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + # Pick your license as you wish + "License :: OSI Approved :: MIT License", + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + # These classifiers are *not* checked by 'pip install'. See instead + # 'python_requires' below. + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], + # This field adds keywords for your project which will appear on the + # project page. What does your project relate to? + # + # Note that this is a string of words separated by whitespace, not a list. + #keywords="sample setuptools development", # Optional + #keywords="sample setuptools development", # Optional + # You can just specify package directories manually here if your project is + # simple. Or you can use find_packages(). + # + # Alternatively, if you just want to distribute a single Python file, use + # the `py_modules` argument instead as follows, which will expect a file + # called `my_module.py` to exist: + # + # py_modules=["my_module"], + # + packages=find_packages(), # Required + # Specify which Python versions you support. In contrast to the + # 'Programming Language' classifiers above, 'pip install' will check this + # and refuse to install the project if the version does not match. If you + # do not support Python 2, you can simplify this to '>=3.5' or similar, see + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4", + # This field lists other packages that your project depends on to run. + # Any package you put here will be installed by pip when your project is + # installed, so they must be valid existing projects. + # + # For an analysis of "install_requires" vs pip's requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + "appdirs==1.4.4", + "attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "black==19.10b0; python_version >= '3.6'", + "cached-property==1.5.2", + "cerberus==1.3.2", + "certifi==2020.12.5", + "chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "distlib==0.3.1", + "idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "jsonschema==3.2.0", + "orderedmultidict==1.0.1", + "packaging==20.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pathspec==0.8.1", + "pep517==0.9.1", + "pip-shims==0.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "pipenv-setup==3.1.1", + "pipfile==0.0.2", + "plette[validation]==0.2.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pyrsistent==0.17.3; python_version >= '3.5'", + "python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "regex==2020.11.13", + "requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "requirementslib==1.5.16; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "tomlkit==0.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "typed-ast==1.4.1", + "urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "vistir==0.5.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "wheel==0.36.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + ], # Optional + # List additional groups of dependencies here (e.g. development + # dependencies). Users will be able to install these using the "extras" + # syntax, for example: + # + # $ pip install sampleproject[dev] + # + # Similar to `install_requires` above, these must be valid existing + # projects. + extras_require={"dev": []}, # Optional + # If there are data files included in your packages that need to be + # installed, specify them here. + # + # Sometimes you’ll want to use packages that are properly arranged with + # setuptools, but are not published to PyPI. In those cases, you can specify + # a list of one or more dependency_links URLs where the package can + # be downloaded, along with some additional hints, and setuptools + # will find and install the package correctly. + # see https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi + # + dependency_links=[], + # If using Python 2.6 or earlier, then these have to be included in + # MANIFEST.in as well. + # package_data={"sample": ["package_data.dat"]}, # Optional + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. See: + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files + # + # In this case, 'data_file' will be installed into '/my_data' + # data_files=[("my_data", ["data/data_file"])], # Optional + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # `pip` to create the appropriate form of executable for the target + # platform. + # + # For example, the following would provide a command called `sample` which + # executes the function `main` from this package when invoked: + # entry_points={"console_scripts": ["sample=sample:main"]}, # Optional + # List additional URLs that are relevant to your project as a dict. + # + # This field corresponds to the "Project-URL" metadata fields: + # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use + # + # Examples listed include a pattern for specifying where the package tracks + # issues, where the source is hosted, where to say thanks to the package + # maintainers, and where to support the project financially. The key is + # what's used to render the link text on PyPI. + #project_urls={ # Optional + # "Bug Reports": "https://github.com/pypa/sampleproject/issues", + # "Funding": "https://donate.pypi.org", + # "Say Thanks!": "http://saythanks.io/to/example", + # "Source": "https://github.com/pypa/sampleproject/", + #}, +) diff --git a/FCRgendata/src/__init__.py b/FCRgendata/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FCRgendata/src/main.py b/FCRgendata/src/main.py new file mode 100644 index 0000000..7e6f9a4 --- /dev/null +++ b/FCRgendata/src/main.py @@ -0,0 +1,30 @@ +""" Generates data using CP-solver and FCR time series for network training """ + +from pathlib import Path +import xml.etree.ElementTree as ET +import numpy as np +import sys +import logging +from .validate_config import validate_config + + +logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) +CONFIG_PATH: Path = None + +if len(sys.argv) != 2: + logging.critical(f"received {len(sys.argv) - 1} arguments, required one: config file path.") + sys.exit(1) +else: + CONFIG_PATH = Path(sys.argv[1]) + + +j_conf = validate_config(CONFIG_PATH) +avg_rsp_times = np.genfromtxt( + j_conf['AvgResponseTimeTableFilePath'], + delimiter=',', + dtype=np.float64, + skip_header=1, + usecols=(6) # read only sixth column (value) +).reshape((-1, 1)) # shape into (n, 1) + +print(avg_rsp_times.shape) diff --git a/FCRgendata/src/validate_config.py b/FCRgendata/src/validate_config.py new file mode 100644 index 0000000..861751b --- /dev/null +++ b/FCRgendata/src/validate_config.py @@ -0,0 +1,79 @@ +"""config validation module + +Defines validate_config function responsible for validating path to config file, +it's structure and content +""" + +from pathlib import Path +import logging +import json +import jsonschema +import sys + + +config_schema = { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "applicationId": {"type": "string"}, + "camelModelFilePath" : {"type": "filepath"}, + "cpProblemFilePath": {"type": "filepath"}, + "nodeCandidatesFilePath": {"type": "filepath"}, + "watermark": { + "type": "object", + "properties": { + "user": {"type": "string"}, + "system": {"type": "string"}, + "date": {"type": "string"}, + "uuid": {"type": "string"}, + }, + "minProperties": 4 + } + }, + "minProperties": 5 + }, + "AvgResponseTimeTableFilePath": {"type": "filepath"}, + "cpSolverHost": {"type": "string"}, # for example "localhost:8080" + "outpath": {"type": "creatablepath"}, + }, + "required": ["request", "AvgResponseTimeTableFilePath"] +} + +# create two custom types: "filepath" (path to exsisting file) and "creatablepath" (path in which new file can be created) +type_checker = jsonschema.Draft3Validator.TYPE_CHECKER.redefine_many({ + "filepath": lambda checker, path: Path(path).is_file(), + "creatablepath": lambda checker, path: Path(path).parent.is_dir() +}) +customValidator = jsonschema.validators.extend(jsonschema.Draft3Validator, type_checker=type_checker) +validator = customValidator(schema=config_schema) + + +def validate_config(conf_p: Path): + """ validates given config path, config path structure and it content, + returns validated json object + """ + + if not conf_p.is_file(): + logging.critical(f"given path: {conf_p} doesn't lead to file") + sys.exit(1) + + j_conf = None + with conf_p.open() as f: + try: + j_conf = json.load(f) + except json.decoder.JSONDecodeError as err: + logging.critical(f"error decoding json config: {err}") + sys.exit(1) + + try: + validator.validate(j_conf) + except jsonschema.exceptions.ValidationError as e: + logging.critical(f'json validation error: {e.message}') + sys.exit(1) + except jsonschema.exceptions.SchemaError as e: + logging.critical(f'json schema error: {e.message}') + sys.exit(1) + + return j_conf -- GitLab From def03901df0abd858ce23c0900ef553fc0268f46 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Wed, 30 Dec 2020 17:06:00 +0100 Subject: [PATCH 03/26] finished state of script, seems to work fine --- FCRgendata/Pipfile | 2 + FCRgendata/Pipfile.lock | 52 +++++++++++++++++- FCRgendata/readme.md | 46 ++++++++++++++++ .../src/__pycache__/__init__.cpython-38.pyc | Bin 0 -> 161 bytes .../src/__pycache__/main.cpython-38.pyc | Bin 0 -> 1531 bytes .../src/__pycache__/utils.cpython-38.pyc | Bin 0 -> 775 bytes .../validate_config.cpython-38.pyc | Bin 0 -> 2180 bytes FCRgendata/src/main.py | 33 +++++++++-- FCRgendata/src/solv_summoner.py | 52 ++++++++++++++++++ FCRgendata/src/validate_config.py | 2 +- 10 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 FCRgendata/readme.md create mode 100644 FCRgendata/src/__pycache__/__init__.cpython-38.pyc create mode 100644 FCRgendata/src/__pycache__/main.cpython-38.pyc create mode 100644 FCRgendata/src/__pycache__/utils.cpython-38.pyc create mode 100644 FCRgendata/src/__pycache__/validate_config.cpython-38.pyc create mode 100644 FCRgendata/src/solv_summoner.py diff --git a/FCRgendata/Pipfile b/FCRgendata/Pipfile index 1b13b8c..782f714 100644 --- a/FCRgendata/Pipfile +++ b/FCRgendata/Pipfile @@ -10,6 +10,8 @@ jsonschema = "*" setuptools = "*" pipenv-setup = "*" numpy = "*" +lxml = "*" +progress = "*" [requires] python_version = "3.8" diff --git a/FCRgendata/Pipfile.lock b/FCRgendata/Pipfile.lock index 2770d65..590980c 100644 --- a/FCRgendata/Pipfile.lock +++ b/FCRgendata/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "555331b6680f90953e06c955358095d7cf2a7f4f74b3fbd998491149e5ea7a16" + "sha256": "3ceafb48a7bc77840c0a82fa99004106b69e316ec11f6ba956adb538a5b1bd4c" }, "pipfile-spec": 6, "requires": { @@ -106,6 +106,49 @@ "index": "pypi", "version": "==3.2.0" }, + "lxml": { + "hashes": [ + "sha256:0448576c148c129594d890265b1a83b9cd76fd1f0a6a04620753d9a6bcfd0a4d", + "sha256:127f76864468d6630e1b453d3ffbbd04b024c674f55cf0a30dc2595137892d37", + "sha256:1471cee35eba321827d7d53d104e7b8c593ea3ad376aa2df89533ce8e1b24a01", + "sha256:2363c35637d2d9d6f26f60a208819e7eafc4305ce39dc1d5005eccc4593331c2", + "sha256:2e5cc908fe43fe1aa299e58046ad66981131a66aea3129aac7770c37f590a644", + "sha256:2e6fd1b8acd005bd71e6c94f30c055594bbd0aa02ef51a22bbfa961ab63b2d75", + "sha256:366cb750140f221523fa062d641393092813b81e15d0e25d9f7c6025f910ee80", + "sha256:42ebca24ba2a21065fb546f3e6bd0c58c3fe9ac298f3a320147029a4850f51a2", + "sha256:4e751e77006da34643ab782e4a5cc21ea7b755551db202bc4d3a423b307db780", + "sha256:4fb85c447e288df535b17ebdebf0ec1cf3a3f1a8eba7e79169f4f37af43c6b98", + "sha256:50c348995b47b5a4e330362cf39fc503b4a43b14a91c34c83b955e1805c8e308", + "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf", + "sha256:535f067002b0fd1a4e5296a8f1bf88193080ff992a195e66964ef2a6cfec5388", + "sha256:5be4a2e212bb6aa045e37f7d48e3e1e4b6fd259882ed5a00786f82e8c37ce77d", + "sha256:60a20bfc3bd234d54d49c388950195d23a5583d4108e1a1d47c9eef8d8c042b3", + "sha256:648914abafe67f11be7d93c1a546068f8eff3c5fa938e1f94509e4a5d682b2d8", + "sha256:681d75e1a38a69f1e64ab82fe4b1ed3fd758717bed735fb9aeaa124143f051af", + "sha256:68a5d77e440df94011214b7db907ec8f19e439507a70c958f750c18d88f995d2", + "sha256:69a63f83e88138ab7642d8f61418cf3180a4d8cd13995df87725cb8b893e950e", + "sha256:6e4183800f16f3679076dfa8abf2db3083919d7e30764a069fb66b2b9eff9939", + "sha256:6fd8d5903c2e53f49e99359b063df27fdf7acb89a52b6a12494208bf61345a03", + "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d", + "sha256:7a7669ff50f41225ca5d6ee0a1ec8413f3a0d8aa2b109f86d540887b7ec0d72a", + "sha256:7e9eac1e526386df7c70ef253b792a0a12dd86d833b1d329e038c7a235dfceb5", + "sha256:7ee8af0b9f7de635c61cdd5b8534b76c52cd03536f29f51151b377f76e214a1a", + "sha256:8246f30ca34dc712ab07e51dc34fea883c00b7ccb0e614651e49da2c49a30711", + "sha256:8c88b599e226994ad4db29d93bc149aa1aff3dc3a4355dd5757569ba78632bdf", + "sha256:923963e989ffbceaa210ac37afc9b906acebe945d2723e9679b643513837b089", + "sha256:94d55bd03d8671686e3f012577d9caa5421a07286dd351dfef64791cf7c6c505", + "sha256:97db258793d193c7b62d4e2586c6ed98d51086e93f9a3af2b2034af01450a74b", + "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f", + "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc", + "sha256:d8d3d4713f0c28bdc6c806a278d998546e8efc3498949e3ace6e117462ac0a5e", + "sha256:e0bfe9bb028974a481410432dbe1b182e8191d5d40382e5b8ff39cdd2e5c5931", + "sha256:f4822c0660c3754f1a41a655e37cb4dbbc9be3d35b125a37fab6f82d47674ebc", + "sha256:f83d281bb2a6217cd806f4cf0ddded436790e66f393e124dfe9731f6b3fb9afe", + "sha256:fc37870d6716b137e80d19241d0e2cff7a7643b925dfa49b4c8ebd1295eb506e" + ], + "index": "pypi", + "version": "==4.6.2" + }, "numpy": { "hashes": [ "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db", @@ -208,6 +251,13 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.3" }, + "progress": { + "hashes": [ + "sha256:69ecedd1d1bbe71bf6313d88d1e6c4d2957b7f1d4f71312c211257f7dae64372" + ], + "index": "pypi", + "version": "==1.5" + }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", diff --git a/FCRgendata/readme.md b/FCRgendata/readme.md new file mode 100644 index 0000000..c491654 --- /dev/null +++ b/FCRgendata/readme.md @@ -0,0 +1,46 @@ +# Training data generator for FCR app +# How to install dependencies + - make sure you have python3 installed + - if you don't have pipenv already installed run: +```sh +$ pip install pipenv +``` + - now run in directory with `Pipfile` +```sh +$ pipenv install +``` +# How to run +### 1. Create config.json + - create `config.json` + - `request` object field must be same as one send to cp-solver to `\constraintProblemSolutionFromFile` endpoint + - `cpSolverHost` string field must represent, web address to working cp-solver instance + - `outpath` string field must represent, path for file with solution to be created + - `AvgResponseTimeTableFilePath` string field must be path to file with FCR time series + +example of `config.json`: +```json +{ + "request": { + "applicationId": "FCRwithDLMSApp", + "camelModelFilePath" : "/home/FCR-data/FCR-model.xmi", + "cpProblemFilePath": "/home/FCR-data/FCR-CP.xmi", + "nodeCandidatesFilePath": "/home/FCR-data/FCR-NodeCandidates", + "watermark": { + "user": "mrozanska", + "system": "UI", + "date": "2017-11-23T16: 41: 41+0000", + "uuid": "fb6280ec-1ab8-11e7-93ae-92361f002671" + } + }, + "cpSolverHost": "localhost:2137", + "outpath": "generated_data.csv", + "AvgResponseTimeTableFilePath": "/home/FCR-time-series/AvgResponseTimeTable.csv" +} +``` + +### 2. Run script + - in directory with `Pipfile` run: +```sh +$ pipenv shell +$ python -m src.main +``` \ No newline at end of file diff --git a/FCRgendata/src/__pycache__/__init__.cpython-38.pyc b/FCRgendata/src/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a3cab149fed18b0ca5c7e65f35a7a52f73607be GIT binary patch literal 161 zcmWIL<>g`kf;}v+<3aRe5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HUenx(7s(x`* zWpQGPer{%NX}P{}QGT|5R6u}!VopIuqHb|XVv%l9YC*ofn{!ZlYFahy%k)K(K)hdvHLS|oMj#D_Yrt)KzhV8JX$^4i=- zNJ_SK4bYp>qSvB;eC)M9XRkdO{Riox9m-8>^w?eOa5&@)XTEQ^&ug`5g3kv({vK|Z z3Hi-V&OcK)d5E9>juP^m2*QY2aH5e?L@ zV&=2r!m&5LCYmRdEkJdA9oH}7`do{kWf2y~H^ikA`YmO_*$TS}uDC3&9MkUwy9Enh zke4smZ3qNyk@0(C9`3;HH}nLnfW^J{;nv=gXo;)h+HvX3A{PFgi|cSx%wdfcxQWph zFvC$zZqvfu;$|1e<|a=-bRo$z#nsVzEa4QH zw_6%RE@M0m!YE67aUV@{QGRw{sG< z9tk&vH`!8D*Pd;zZ#)UN9(A9dI(X7(Nu!Nw5Xj(%#a0r`X@5?XoEWGEQOxih05u<-n?X zKhDEl-~w%xfrcoPnCj`D_$U{>Sj8R=+OP{zzNIn@oV<%X3Znc=Fe;ApHxFa+>C%7F zSb8}ssk_EvoMZwD4`CcYx9}eAtZi(x9o9HZWGEn#90^7I3Lnxa6UfeJ20h|l3}O}c zjMh54SppqBI?`Npk~kS0bUswseur&sb-2uT`HD7NttiN|4zk~e)HX-wd=NGhMP~gzR>-QaxQ?T>NkcJrV{`qPvcH_75r#ZX zGc!R%V|pbCY2ji!44EIP*u)W+g$swVVdcF^N3v-wxTmtj92!>X$&8y%KC;Z|U5+Hz z_^=-;orm^LXy)ZyX<$C$6y;b9dc+8qlx+6=#?UU6vAeMW3a6Jd|6K?y1PltJ3hw|D zg2J`^{tHoy7BMT{e*Qlh(<`qjQ7$`gUB8JdKKl4{jyCBmj&=GMC@;ta literal 0 HcmV?d00001 diff --git a/FCRgendata/src/__pycache__/utils.cpython-38.pyc b/FCRgendata/src/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c5277bf5ce0e22423a957be5b34f8de5f4141b3 GIT binary patch literal 775 zcmYjPJ8u**5MFy9cX^2*A!>^%E9F=L9YP3kQp)H=pxNeR*1qf>yS5p76Wu8kNVMS( z08&!&m)ufC`~xUpyq6+ld0raNHxIwt+uM$iPQU)1|0PlMD;RevM&lJ;`x=>u1Zks^ z43d^~kZ~CgGAR?8EK@;6d`ZeojwD%*Me?1BRAgV0a!ZhtC?DNmp6K`{L7o$zHA+b3 za_V=iZqFplYi*s+K)N1OMMLl{g-TMqQ@r*mG8Zk0AQ$9YB z8GoWKxk@kSGP@=u5?MfBe`XbLTgxgl*#NSIhF}@c?_1e1As3C3f;EbrHfK_?Mbk>7 z8|5UhgXwHx^22R*OPew+Q|2)T;460CxSH9T8_D#79jHV0Bxl?*khX8#@NPY-oiopi zV*RM7w6YF3CZBEKPjuV6Myn&x-4Oz>cc&VWm7Ba>8M)r)#xC&HY<2*YCf@!!vt_j#nJT5sZ^n&!a`MGFRguS zR{lX{Zubk-9JQgzt+2`A`=t*Fjp)B|4ENEOZ~IhXu`5rNfiZd@gu*T|zc<67yVWc- oAOUuQFUBGIblAX}2(3X?%SvtcQ1yaPRg|PEpoWl?e#~X%cxE@=_=_2D z+N?Ge5*3^(xHpwH$NozrZU~78i3=A52=QjTn}kZ0u;u4B^WOW-o98z_e_5-!2(I_P z`?Vc__LCl*ys9v`4UhZ<1Va=t%&Z8<7PYX}?Z~FKc@s*^yF^Ro?NEmixJ%3lG05^8 z^4LOdP|c>t%kv<>je2 z2?-;9)eZA2(v!L92YFv`&rg`P=RC>XL#R4xVnx;;#Vswy3T z(a24rNMCXRmR*_iSP@8_Q_}B;?2u?ATj-Z^Gm&Oo*z$(Y>+_+>@S{7(S-y&mLPXsE-sJer=Gx2#J<=*Ea|0df#81b zL+m)?vfy9!xy;q#$2;BiqICRz81wtuX@aAwLAH@bJ6zmNWlqa!KiBc10hnS&i&+rk z{fTMNX+;Bvg0sMa;pEb~ewN^oy1R2U#NXf%K0q7jHC}C6LYq}Nlck6iMr(S3mR8PzfQ>s^*SOGeRo!Ck@y?r=Iui8Q2aBVdx19wT z94D9lw7Te~s+YhrL?d*7ABR60Q4KiIPd_1qbi zhj3i?@I#c`CMb9Gf!=n-*BI^D7!58Mw9?RiiC%%6tIk zI(mcv*&R&}v5o-b=YR}k(5nOTkAO@7*)zz_kn|dF#U{^)!CoH6&uqItU~%C;2<88U z0ND*O&`f2JOvnNFq#N$=WWo%()2^Nzyg?9QKH-R~t~b5}2mVNoiHUhV3A|NshVdFy zi$yg0hu+|{$5JknE4deOpJ`sw9e=1l?CM~V3jyTCc#twpT?VV*EeJhhh9}t3c!S}T zF%-?>jVGJ4;uK)2a+G$vhOq(>=0Gu#vVo}-;ZH*#HK3$K5go5cl=@7S&Ej0p#-|&f zuP*CM_=?U-O^sJh1+7dDlX7)vv9fac(|~81jwNk8I3n>Px*T&U{Vu1|8>XqvY_lZZ z)k!)%L+A>3lnwbS%-=XRsxet{%=|YYOkM_oN)Eo3|0IAZ|VM!cR&!QiYvuUEZediY;NmF`%!{|g+ERc8PI literal 0 HcmV?d00001 diff --git a/FCRgendata/src/main.py b/FCRgendata/src/main.py index 7e6f9a4..49ddff8 100644 --- a/FCRgendata/src/main.py +++ b/FCRgendata/src/main.py @@ -1,11 +1,16 @@ """ Generates data using CP-solver and FCR time series for network training """ from pathlib import Path -import xml.etree.ElementTree as ET +from .validate_config import validate_config +from .solv_summoner import CPSolverSolutionSummoner +from progress.bar import IncrementalBar as IBar +from contextlib import ExitStack import numpy as np import sys import logging -from .validate_config import validate_config +import json +import os +import csv logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) @@ -19,12 +24,30 @@ else: j_conf = validate_config(CONFIG_PATH) -avg_rsp_times = np.genfromtxt( +avg_rsp_times = np.genfromtxt( # shape (n, ) j_conf['AvgResponseTimeTableFilePath'], delimiter=',', dtype=np.float64, skip_header=1, usecols=(6) # read only sixth column (value) -).reshape((-1, 1)) # shape into (n, 1) +) + +solver_host = j_conf['cpSolverHost'] +csv_header = [ 'AvgResponseTime', 'cardinality_Component_LB', 'provider_Component_LB', + 'AppCardinality', 'cardinality_Component_DB', 'provider_Component_App', + 'provider_Component_DB'] + +with ExitStack() as stack: # substiture for multiline with statement + summoner = stack.enter_context(CPSolverSolutionSummoner(j_conf)) + bar = stack.enter_context(IBar('Progress', max=len(avg_rsp_times))) + csvfile = stack.enter_context(open(j_conf['outpath'], 'w', newline='')) + + writer = csv.writer(csvfile, delimiter=',') + writer.writerow(csv_header) + + for rsp_time in avg_rsp_times: + solution = summoner.get_solution(rsp_time) + writer.writerow([rsp_time, *solution]) + bar.next() + -print(avg_rsp_times.shape) diff --git a/FCRgendata/src/solv_summoner.py b/FCRgendata/src/solv_summoner.py new file mode 100644 index 0000000..557f8ef --- /dev/null +++ b/FCRgendata/src/solv_summoner.py @@ -0,0 +1,52 @@ +from pathlib import Path +from lxml import etree +from tempfile import NamedTemporaryFile +from contextlib import ContextDecorator +from typing import List +from functools import lru_cache +import requests +import json +import os + + +CACHE_SIZE = 2**10 + + +class CPSolverSolutionSummoner(ContextDecorator): + """ calls cp-solver for solution, and manages additional resources """ + + def __init__(self, config): + self.solver_host = config['cpSolverHost'] + self.solution_file = f"{config['request']['camelModelFilePath'][:-4]}-solution.xmi" + self.FCRcpxml = etree.parse(config['request']['cpProblemFilePath']) + self.avg_rsp_time_xelem, = self.FCRcpxml.find("cpMetrics[@id='AvgResponseTime']") + self.req = config['request'] + + def __enter__(self): + self.tempfile = NamedTemporaryFile() + self.req['cpProblemFilePath'] = self.tempfile.name + return self + + def __exit__(self, *exc): + self.tempfile.close() + if os.path.exists(self.solution_file): + os.remove(self.solution_file) + return False + + @lru_cache(maxsize=CACHE_SIZE) + def get_solution(self, avg_rsp_time: float) -> List[int]: + """ returns solution as list of int """ + + self.avg_rsp_time_xelem.set('value', str(avg_rsp_time)) + self.FCRcpxml.write(self.tempfile.name, xml_declaration=True, encoding="ASCII") + r = requests.post(f'http://{self.solver_host}/constraintProblemSolutionFromFile', + data=json.dumps(self.req), + headers={'Content-Type': 'application/json'} + ) + + if r.status_code != 200: + raise Exception("cp-solver returned non 200 status code") + + sol_xelem = etree.parse(self.solution_file).find("solution") + return [int(v.get('value', 0)) for v in sol_xelem.iterfind(".//value")] + diff --git a/FCRgendata/src/validate_config.py b/FCRgendata/src/validate_config.py index 861751b..b9893fc 100644 --- a/FCRgendata/src/validate_config.py +++ b/FCRgendata/src/validate_config.py @@ -38,7 +38,7 @@ config_schema = { "cpSolverHost": {"type": "string"}, # for example "localhost:8080" "outpath": {"type": "creatablepath"}, }, - "required": ["request", "AvgResponseTimeTableFilePath"] + "required": ["request", "AvgResponseTimeTableFilePath", "cpSolverHost", "outpath"] } # create two custom types: "filepath" (path to exsisting file) and "creatablepath" (path in which new file can be created) -- GitLab From 426294bc35c7843fcf9c730e0baf4be3251301a0 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Wed, 30 Dec 2020 18:04:34 +0100 Subject: [PATCH 04/26] style refractor --- FCRgendata/config.json | 8 +++++--- FCRgendata/src/main.py | 15 ++++++--------- FCRgendata/src/validate_config.py | 10 ++++++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/FCRgendata/config.json b/FCRgendata/config.json index 7fbfc5e..c9a0b27 100644 --- a/FCRgendata/config.json +++ b/FCRgendata/config.json @@ -1,9 +1,9 @@ { "request": { "applicationId": "FCRwithDLMSApp", - "camelModelFilePath" : "/home/szysad/mimuw/3rok/ZPP/FCR-problems/podobno-dane-co-dzialaja/FCRnewWithSensor2CoresOneInstance.xmi", - "cpProblemFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/podobno-dane-co-dzialaja/FCR-CP.xmi", - "nodeCandidatesFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/podobno-dane-co-dzialaja/FCR-NodeCandidates", + "camelModelFilePath" : "/home/szysad/mimuw/3rok/ZPP/FCR-problems/FRC-dane-treningowe/FCR-model.xmi", + "cpProblemFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/FRC-dane-treningowe/FCR-CP.xmi", + "nodeCandidatesFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/FRC-dane-treningowe/FCR-NodeCandidates", "watermark": { "user": "mrozanska", "system": "UI", @@ -11,5 +11,7 @@ "uuid": "fb6280ec-1ab8-11e7-93ae-92361f002671" } }, + "cpSolverHost": "localhost:8080", + "outpath": "generated_data.csv", "AvgResponseTimeTableFilePath": "/home/szysad/mimuw/3rok/ZPP/time-series-data/time-series-data/secure-document​/deployment-reconfiguration-range-2-to-2/2020-10-30 to 2020-10-30/V 1.0 - raw data/AvgResponseTimeTable.csv" } \ No newline at end of file diff --git a/FCRgendata/src/main.py b/FCRgendata/src/main.py index 49ddff8..c029ec6 100644 --- a/FCRgendata/src/main.py +++ b/FCRgendata/src/main.py @@ -8,8 +8,6 @@ from contextlib import ExitStack import numpy as np import sys import logging -import json -import os import csv @@ -17,7 +15,8 @@ logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) CONFIG_PATH: Path = None if len(sys.argv) != 2: - logging.critical(f"received {len(sys.argv) - 1} arguments, required one: config file path.") + logging.critical( + f"received {len(sys.argv) - 1} arguments, required one: config file path.") sys.exit(1) else: CONFIG_PATH = Path(sys.argv[1]) @@ -29,13 +28,13 @@ avg_rsp_times = np.genfromtxt( # shape (n, ) delimiter=',', dtype=np.float64, skip_header=1, - usecols=(6) # read only sixth column (value) + usecols=(6) # read only sixth column (value) ) solver_host = j_conf['cpSolverHost'] -csv_header = [ 'AvgResponseTime', 'cardinality_Component_LB', 'provider_Component_LB', - 'AppCardinality', 'cardinality_Component_DB', 'provider_Component_App', - 'provider_Component_DB'] +csv_header = ['AvgResponseTime', 'cardinality_Component_LB', 'provider_Component_LB', + 'AppCardinality', 'cardinality_Component_DB', 'provider_Component_App', + 'provider_Component_DB'] with ExitStack() as stack: # substiture for multiline with statement summoner = stack.enter_context(CPSolverSolutionSummoner(j_conf)) @@ -49,5 +48,3 @@ with ExitStack() as stack: # substiture for multiline with statement solution = summoner.get_solution(rsp_time) writer.writerow([rsp_time, *solution]) bar.next() - - diff --git a/FCRgendata/src/validate_config.py b/FCRgendata/src/validate_config.py index b9893fc..1bf5553 100644 --- a/FCRgendata/src/validate_config.py +++ b/FCRgendata/src/validate_config.py @@ -18,7 +18,7 @@ config_schema = { "type": "object", "properties": { "applicationId": {"type": "string"}, - "camelModelFilePath" : {"type": "filepath"}, + "camelModelFilePath": {"type": "filepath"}, "cpProblemFilePath": {"type": "filepath"}, "nodeCandidatesFilePath": {"type": "filepath"}, "watermark": { @@ -35,18 +35,20 @@ config_schema = { "minProperties": 5 }, "AvgResponseTimeTableFilePath": {"type": "filepath"}, - "cpSolverHost": {"type": "string"}, # for example "localhost:8080" + "cpSolverHost": {"type": "string"}, # for example "localhost:8080" "outpath": {"type": "creatablepath"}, }, "required": ["request", "AvgResponseTimeTableFilePath", "cpSolverHost", "outpath"] } -# create two custom types: "filepath" (path to exsisting file) and "creatablepath" (path in which new file can be created) +# create two custom types: "filepath" (path to exsisting file) +# and "creatablepath" (path in which new file can be created) type_checker = jsonschema.Draft3Validator.TYPE_CHECKER.redefine_many({ "filepath": lambda checker, path: Path(path).is_file(), "creatablepath": lambda checker, path: Path(path).parent.is_dir() }) -customValidator = jsonschema.validators.extend(jsonschema.Draft3Validator, type_checker=type_checker) +customValidator = jsonschema.validators.extend( + jsonschema.Draft3Validator, type_checker=type_checker) validator = customValidator(schema=config_schema) -- GitLab From 24c04be3762cbe93c25b3a32e3e48d261cbaeae7 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Mon, 4 Jan 2021 12:27:06 +0100 Subject: [PATCH 05/26] removed cache files --- .../src/__pycache__/__init__.cpython-38.pyc | Bin 161 -> 0 bytes FCRgendata/src/__pycache__/main.cpython-38.pyc | Bin 1531 -> 0 bytes FCRgendata/src/__pycache__/utils.cpython-38.pyc | Bin 775 -> 0 bytes .../__pycache__/validate_config.cpython-38.pyc | Bin 2180 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 FCRgendata/src/__pycache__/__init__.cpython-38.pyc delete mode 100644 FCRgendata/src/__pycache__/main.cpython-38.pyc delete mode 100644 FCRgendata/src/__pycache__/utils.cpython-38.pyc delete mode 100644 FCRgendata/src/__pycache__/validate_config.cpython-38.pyc diff --git a/FCRgendata/src/__pycache__/__init__.cpython-38.pyc b/FCRgendata/src/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 2a3cab149fed18b0ca5c7e65f35a7a52f73607be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 161 zcmWIL<>g`kf;}v+<3aRe5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HUenx(7s(x`* zWpQGPer{%NX}P{}QGT|5R6u}!VopIuqHb|XVv%l9YC*ofn{!ZlYFahy%k)K(K)hdvHLS|oMj#D_Yrt)KzhV8JX$^4i=- zNJ_SK4bYp>qSvB;eC)M9XRkdO{Riox9m-8>^w?eOa5&@)XTEQ^&ug`5g3kv({vK|Z z3Hi-V&OcK)d5E9>juP^m2*QY2aH5e?L@ zV&=2r!m&5LCYmRdEkJdA9oH}7`do{kWf2y~H^ikA`YmO_*$TS}uDC3&9MkUwy9Enh zke4smZ3qNyk@0(C9`3;HH}nLnfW^J{;nv=gXo;)h+HvX3A{PFgi|cSx%wdfcxQWph zFvC$zZqvfu;$|1e<|a=-bRo$z#nsVzEa4QH zw_6%RE@M0m!YE67aUV@{QGRw{sG< z9tk&vH`!8D*Pd;zZ#)UN9(A9dI(X7(Nu!Nw5Xj(%#a0r`X@5?XoEWGEQOxih05u<-n?X zKhDEl-~w%xfrcoPnCj`D_$U{>Sj8R=+OP{zzNIn@oV<%X3Znc=Fe;ApHxFa+>C%7F zSb8}ssk_EvoMZwD4`CcYx9}eAtZi(x9o9HZWGEn#90^7I3Lnxa6UfeJ20h|l3}O}c zjMh54SppqBI?`Npk~kS0bUswseur&sb-2uT`HD7NttiN|4zk~e)HX-wd=NGhMP~gzR>-QaxQ?T>NkcJrV{`qPvcH_75r#ZX zGc!R%V|pbCY2ji!44EIP*u)W+g$swVVdcF^N3v-wxTmtj92!>X$&8y%KC;Z|U5+Hz z_^=-;orm^LXy)ZyX<$C$6y;b9dc+8qlx+6=#?UU6vAeMW3a6Jd|6K?y1PltJ3hw|D zg2J`^{tHoy7BMT{e*Qlh(<`qjQ7$`gUB8JdKKl4{jyCBmj&=GMC@;ta diff --git a/FCRgendata/src/__pycache__/utils.cpython-38.pyc b/FCRgendata/src/__pycache__/utils.cpython-38.pyc deleted file mode 100644 index 1c5277bf5ce0e22423a957be5b34f8de5f4141b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 775 zcmYjPJ8u**5MFy9cX^2*A!>^%E9F=L9YP3kQp)H=pxNeR*1qf>yS5p76Wu8kNVMS( z08&!&m)ufC`~xUpyq6+ld0raNHxIwt+uM$iPQU)1|0PlMD;RevM&lJ;`x=>u1Zks^ z43d^~kZ~CgGAR?8EK@;6d`ZeojwD%*Me?1BRAgV0a!ZhtC?DNmp6K`{L7o$zHA+b3 za_V=iZqFplYi*s+K)N1OMMLl{g-TMqQ@r*mG8Zk0AQ$9YB z8GoWKxk@kSGP@=u5?MfBe`XbLTgxgl*#NSIhF}@c?_1e1As3C3f;EbrHfK_?Mbk>7 z8|5UhgXwHx^22R*OPew+Q|2)T;460CxSH9T8_D#79jHV0Bxl?*khX8#@NPY-oiopi zV*RM7w6YF3CZBEKPjuV6Myn&x-4Oz>cc&VWm7Ba>8M)r)#xC&HY<2*YCf@!!vt_j#nJT5sZ^n&!a`MGFRguS zR{lX{Zubk-9JQgzt+2`A`=t*Fjp)B|4ENEOZ~IhXu`5rNfiZd@gu*T|zc<67yVWc- oAOUuQFUBGIblAX}2(3X?%SvtcQ1yaPRg|PEpoWl?e#~X%cxE@=_=_2D z+N?Ge5*3^(xHpwH$NozrZU~78i3=A52=QjTn}kZ0u;u4B^WOW-o98z_e_5-!2(I_P z`?Vc__LCl*ys9v`4UhZ<1Va=t%&Z8<7PYX}?Z~FKc@s*^yF^Ro?NEmixJ%3lG05^8 z^4LOdP|c>t%kv<>je2 z2?-;9)eZA2(v!L92YFv`&rg`P=RC>XL#R4xVnx;;#Vswy3T z(a24rNMCXRmR*_iSP@8_Q_}B;?2u?ATj-Z^Gm&Oo*z$(Y>+_+>@S{7(S-y&mLPXsE-sJer=Gx2#J<=*Ea|0df#81b zL+m)?vfy9!xy;q#$2;BiqICRz81wtuX@aAwLAH@bJ6zmNWlqa!KiBc10hnS&i&+rk z{fTMNX+;Bvg0sMa;pEb~ewN^oy1R2U#NXf%K0q7jHC}C6LYq}Nlck6iMr(S3mR8PzfQ>s^*SOGeRo!Ck@y?r=Iui8Q2aBVdx19wT z94D9lw7Te~s+YhrL?d*7ABR60Q4KiIPd_1qbi zhj3i?@I#c`CMb9Gf!=n-*BI^D7!58Mw9?RiiC%%6tIk zI(mcv*&R&}v5o-b=YR}k(5nOTkAO@7*)zz_kn|dF#U{^)!CoH6&uqItU~%C;2<88U z0ND*O&`f2JOvnNFq#N$=WWo%()2^Nzyg?9QKH-R~t~b5}2mVNoiHUhV3A|NshVdFy zi$yg0hu+|{$5JknE4deOpJ`sw9e=1l?CM~V3jyTCc#twpT?VV*EeJhhh9}t3c!S}T zF%-?>jVGJ4;uK)2a+G$vhOq(>=0Gu#vVo}-;ZH*#HK3$K5go5cl=@7S&Ej0p#-|&f zuP*CM_=?U-O^sJh1+7dDlX7)vv9fac(|~81jwNk8I3n>Px*T&U{Vu1|8>XqvY_lZZ z)k!)%L+A>3lnwbS%-=XRsxet{%=|YYOkM_oN)Eo3|0IAZ|VM!cR&!QiYvuUEZediY;NmF`%!{|g+ERc8PI -- GitLab From f1f71143230ea67412fc2b355aff9cc87f110001 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Wed, 6 Jan 2021 09:43:41 +0100 Subject: [PATCH 06/26] adder data generation from AvgResponseTimeTableFile and predictionsFile --- FCRgendata/readme.md | 4 ++- FCRgendata/src/main.py | 48 ++++++++++++++++++++++++------- FCRgendata/src/solv_summoner.py | 24 ++++++++++------ FCRgendata/src/validate_config.py | 3 +- 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/FCRgendata/readme.md b/FCRgendata/readme.md index c491654..8a0f296 100644 --- a/FCRgendata/readme.md +++ b/FCRgendata/readme.md @@ -16,6 +16,7 @@ $ pipenv install - `cpSolverHost` string field must represent, web address to working cp-solver instance - `outpath` string field must represent, path for file with solution to be created - `AvgResponseTimeTableFilePath` string field must be path to file with FCR time series + - `predictionsFilePath` string field must be path to file with FCR avgResponseTime predictions example of `config.json`: ```json @@ -34,7 +35,8 @@ example of `config.json`: }, "cpSolverHost": "localhost:2137", "outpath": "generated_data.csv", - "AvgResponseTimeTableFilePath": "/home/FCR-time-series/AvgResponseTimeTable.csv" + "AvgResponseTimeTableFilePath": "/home/FCR-time-series/AvgResponseTimeTable.csv", + "predictionsFilePath": "/home/FCR-time-series/predictions1.csv" } ``` diff --git a/FCRgendata/src/main.py b/FCRgendata/src/main.py index c029ec6..85882c9 100644 --- a/FCRgendata/src/main.py +++ b/FCRgendata/src/main.py @@ -2,9 +2,10 @@ from pathlib import Path from .validate_config import validate_config -from .solv_summoner import CPSolverSolutionSummoner +from .solv_summoner import CPSolverSolutionSummoner, CPSolverSolutionSummonerError from progress.bar import IncrementalBar as IBar from contextlib import ExitStack +import numpy.lib.recfunctions as rfn import numpy as np import sys import logging @@ -26,25 +27,52 @@ j_conf = validate_config(CONFIG_PATH) avg_rsp_times = np.genfromtxt( # shape (n, ) j_conf['AvgResponseTimeTableFilePath'], delimiter=',', - dtype=np.float64, + dtype=np.dtype([('avg_rsp_time', np.float64), ('datetime', 'datetime64[s]')]), skip_header=1, - usecols=(6) # read only sixth column (value) + usecols=(6, 1) ) +avg_rsp_prediction = np.genfromtxt( # shape (n, ) + j_conf['predictionsFilePath'], + delimiter=',', + dtype=np.dtype([('avg_rsp_time_pred', np.float64), ('datetime', 'datetime64[s]'), ('split', np.int)]), + skip_header=1, + usecols=(1, 3, 4), + converters={4: lambda split: {b"train": 0, b"val": 1, b"test": -1}[split]} +) + + +joined = rfn.join_by('datetime', avg_rsp_times, avg_rsp_prediction, jointype='inner', usemask=False) +assert len(joined) == len(avg_rsp_prediction) +# joined is structured array of tuples with dtype +# [('datetime', ' Date: Wed, 6 Jan 2021 14:39:28 +0100 Subject: [PATCH 07/26] small FCR data set for initial experiments. Currently using data from single file from 2020-10-29 to 2020-10-30 experiment. Wrapped in pytorch dataset. --- FCRdataLoader/Pipfile | 14 + FCRdataLoader/Pipfile.lock | 393 ++++++++++ FCRdataLoader/README.md | 47 ++ FCRdataLoader/data/data1.csv | 998 ++++++++++++++++++++++++ FCRdataLoader/data/data2.csv | 218 ++++++ FCRdataLoader/data/data4.csv | 116 +++ FCRdataLoader/fcrdataloader/__init__.py | 0 FCRdataLoader/fcrdataloader/data.py | 5 + FCRdataLoader/fcrdataloader/dataset.py | 116 +++ FCRdataLoader/setup.py | 156 ++++ 10 files changed, 2063 insertions(+) create mode 100644 FCRdataLoader/Pipfile create mode 100644 FCRdataLoader/Pipfile.lock create mode 100644 FCRdataLoader/README.md create mode 100644 FCRdataLoader/data/data1.csv create mode 100644 FCRdataLoader/data/data2.csv create mode 100644 FCRdataLoader/data/data4.csv create mode 100644 FCRdataLoader/fcrdataloader/__init__.py create mode 100644 FCRdataLoader/fcrdataloader/data.py create mode 100644 FCRdataLoader/fcrdataloader/dataset.py create mode 100644 FCRdataLoader/setup.py diff --git a/FCRdataLoader/Pipfile b/FCRdataLoader/Pipfile new file mode 100644 index 0000000..43457cf --- /dev/null +++ b/FCRdataLoader/Pipfile @@ -0,0 +1,14 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +pipenv-setup = "*" + +[packages] +numpy = "*" +torch = "*" + +[requires] +python_version = "3.8" diff --git a/FCRdataLoader/Pipfile.lock b/FCRdataLoader/Pipfile.lock new file mode 100644 index 0000000..2b43faa --- /dev/null +++ b/FCRdataLoader/Pipfile.lock @@ -0,0 +1,393 @@ +{ + "_meta": { + "hash": { + "sha256": "97a59620fc51f3a45a147fa7068de11884d82d7f6b6516eab9e7ee9b0ce22cbf" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "numpy": { + "hashes": [ + "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94", + "sha256:06fab248a088e439402141ea04f0fffb203723148f6ee791e9c75b3e9e82f080", + "sha256:0eef32ca3132a48e43f6a0f5a82cb508f22ce5a3d6f67a8329c81c8e226d3f6e", + "sha256:1ded4fce9cfaaf24e7a0ab51b7a87be9038ea1ace7f34b841fe3b6894c721d1c", + "sha256:2e55195bc1c6b705bfd8ad6f288b38b11b1af32f3c8289d6c50d47f950c12e76", + "sha256:2ea52bd92ab9f768cc64a4c3ef8f4b2580a17af0a5436f6126b08efbd1838371", + "sha256:36674959eed6957e61f11c912f71e78857a8d0604171dfd9ce9ad5cbf41c511c", + "sha256:384ec0463d1c2671170901994aeb6dce126de0a95ccc3976c43b0038a37329c2", + "sha256:39b70c19ec771805081578cc936bbe95336798b7edf4732ed102e7a43ec5c07a", + "sha256:400580cbd3cff6ffa6293df2278c75aef2d58d8d93d3c5614cd67981dae68ceb", + "sha256:43d4c81d5ffdff6bae58d66a3cd7f54a7acd9a0e7b18d97abb255defc09e3140", + "sha256:50a4a0ad0111cc1b71fa32dedd05fa239f7fb5a43a40663269bb5dc7877cfd28", + "sha256:603aa0706be710eea8884af807b1b3bc9fb2e49b9f4da439e76000f3b3c6ff0f", + "sha256:6149a185cece5ee78d1d196938b2a8f9d09f5a5ebfbba66969302a778d5ddd1d", + "sha256:759e4095edc3c1b3ac031f34d9459fa781777a93ccc633a472a5468587a190ff", + "sha256:7fb43004bce0ca31d8f13a6eb5e943fa73371381e53f7074ed21a4cb786c32f8", + "sha256:811daee36a58dc79cf3d8bdd4a490e4277d0e4b7d103a001a4e73ddb48e7e6aa", + "sha256:8b5e972b43c8fc27d56550b4120fe6257fdc15f9301914380b27f74856299fea", + "sha256:99abf4f353c3d1a0c7a5f27699482c987cf663b1eac20db59b8c7b061eabd7fc", + "sha256:a0d53e51a6cb6f0d9082decb7a4cb6dfb33055308c4c44f53103c073f649af73", + "sha256:a12ff4c8ddfee61f90a1633a4c4afd3f7bcb32b11c52026c92a12e1325922d0d", + "sha256:a4646724fba402aa7504cd48b4b50e783296b5e10a524c7a6da62e4a8ac9698d", + "sha256:a76f502430dd98d7546e1ea2250a7360c065a5fdea52b2dffe8ae7180909b6f4", + "sha256:a9d17f2be3b427fbb2bce61e596cf555d6f8a56c222bd2ca148baeeb5e5c783c", + "sha256:ab83f24d5c52d60dbc8cd0528759532736b56db58adaa7b5f1f76ad551416a1e", + "sha256:aeb9ed923be74e659984e321f609b9ba54a48354bfd168d21a2b072ed1e833ea", + "sha256:c843b3f50d1ab7361ca4f0b3639bf691569493a56808a0b0c54a051d260b7dbd", + "sha256:cae865b1cae1ec2663d8ea56ef6ff185bad091a5e33ebbadd98de2cfa3fa668f", + "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff", + "sha256:cf2402002d3d9f91c8b01e66fbb436a4ed01c6498fffed0e4c7566da1d40ee1e", + "sha256:d051ec1c64b85ecc69531e1137bb9751c6830772ee5c1c426dbcfe98ef5788d7", + "sha256:d6631f2e867676b13026e2846180e2c13c1e11289d67da08d71cacb2cd93d4aa", + "sha256:dbd18bcf4889b720ba13a27ec2f2aac1981bd41203b3a3b27ba7a33f88ae4827", + "sha256:df609c82f18c5b9f6cb97271f03315ff0dbe481a2a02e56aeb1b1a985ce38e60" + ], + "index": "pypi", + "version": "==1.19.5" + }, + "torch": { + "hashes": [ + "sha256:2e49cac969976be63117004ee00d0a3e3dd4ea662ad77383f671b8992825de1a", + "sha256:38d67f4fb189a92a977b2c0a38e4f6dd413e0bf55aa6d40004696df7e40a71ff", + "sha256:422e64e98d0e100c360993819d0307e5d56e9517b26135808ad68984d577d75a", + "sha256:5d76c255a41484c1d41a9ff570b9c9f36cb85df9428aa15a58ae16ac7cfc2ea6", + "sha256:6652a767a0572ae0feb74ad128758e507afd3b8396b6e7f147e438ba8d4c6f63", + "sha256:a3793dcceb12b1e2281290cca1277c5ce86ddfd5bf044f654285a4d69057aea7", + "sha256:af464a6f4314a875035e0c4c2b07517599704b214634f4ed3ad2e748c5ef291f", + "sha256:d241c3f1c4d563e4ba86f84769c23e12606db167ee6f674eedff6d02901462e3", + "sha256:dd2fc6880c95e836960d86efbbc7f63d3287f2e1893c51d31f96dbfe02f0d73e", + "sha256:de84b4166e3f7335eb868b51d3bbd909ec33828af27290b4171bce832a55be3c", + "sha256:e000b94be3aa58ad7f61e7d07cf379ea9366cf6c6874e68bd58ad0bdc537b3a7", + "sha256:f0aaf657145533824b15f2fd8fde8f8c67fe6c6281088ef588091f03fad90243" + ], + "index": "pypi", + "version": "==1.7.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + } + }, + "develop": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.3.0" + }, + "black": { + "hashes": [ + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" + ], + "markers": "python_version >= '3.6'", + "version": "==19.10b0" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "cerberus": { + "hashes": [ + "sha256:302e6694f206dd85cb63f13fd5025b31ab6d38c99c50c6d769f8fa0b0f299589" + ], + "version": "==1.3.2" + }, + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.4" + }, + "distlib": { + "hashes": [ + "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", + "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" + ], + "version": "==0.3.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "orderedmultidict": { + "hashes": [ + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" + ], + "version": "==1.0.1" + }, + "packaging": { + "hashes": [ + "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858", + "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.8" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, + "pep517": { + "hashes": [ + "sha256:3985b91ebf576883efe5fa501f42a16de2607684f3797ddba7202b71b7d0da51", + "sha256:aeb78601f2d1aa461960b43add204cc7955667687fbcf9cdb5170f00556f117f" + ], + "version": "==0.9.1" + }, + "pip-shims": { + "hashes": [ + "sha256:05b00ade9d1e686a98bb656dd9b0608a933897283dc21913fad6ea5409ff7e91", + "sha256:16ca9f87485667b16b978b68a1aae4f9cc082c0fa018aed28567f9f34a590569" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.5.3" + }, + "pipenv-setup": { + "hashes": [ + "sha256:8a439aff7b16e18d7e07702c9186fc5fe86156679eace90e10c2578a43bd7af1", + "sha256:e1bfd55c1152024e762f1c17f6189fcb073166509e7c0228870f7ea160355648" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pipfile": { + "hashes": [ + "sha256:f7d9f15de8b660986557eb3cc5391aa1a16207ac41bc378d03f414762d36c984" + ], + "version": "==0.0.2" + }, + "plette": { + "extras": [ + "validation" + ], + "hashes": [ + "sha256:46402c03e36d6eadddad2a5125990e322dd74f98160c8f2dcd832b2291858a26", + "sha256:d6c9b96981b347bddd333910b753b6091a2c1eb2ef85bb373b4a67c9d91dca16" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.3" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.1" + }, + "regex": { + "hashes": [ + "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", + "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", + "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", + "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", + "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", + "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", + "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", + "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", + "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", + "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", + "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", + "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", + "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", + "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", + "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", + "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", + "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", + "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", + "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", + "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", + "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", + "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", + "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", + "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", + "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", + "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", + "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", + "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", + "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", + "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", + "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", + "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", + "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", + "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", + "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", + "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", + "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", + "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", + "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", + "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", + "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + ], + "version": "==2020.11.13" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.25.1" + }, + "requirementslib": { + "hashes": [ + "sha256:50d20f27e4515a2393695b0d886219598302163438ae054253147b2bad9b4a44", + "sha256:9c1e8666ca4512724cdd1739adcc7df19ec7ad2ed21f0e748f9631ad6b54f321" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.5.16" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "tomlkit": { + "hashes": [ + "sha256:6babbd33b17d5c9691896b0e68159215a9387ebfa938aa3ac42f4a4beeb2b831", + "sha256:ac57f29693fab3e309ea789252fcce3061e19110085aa31af5446ca749325618" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.7.0" + }, + "typed-ast": { + "hashes": [ + "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1", + "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d", + "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6", + "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd", + "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37", + "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151", + "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07", + "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440", + "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70", + "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496", + "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea", + "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400", + "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc", + "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606", + "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc", + "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581", + "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412", + "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a", + "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2", + "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787", + "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f", + "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937", + "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64", + "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487", + "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b", + "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41", + "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a", + "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3", + "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166", + "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10" + ], + "version": "==1.4.2" + }, + "urllib3": { + "hashes": [ + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.2" + }, + "vistir": { + "hashes": [ + "sha256:a37079cdbd85d31a41cdd18457fe521e15ec08b255811e81aa061fd5f48a20fb", + "sha256:eff1d19ef50c703a329ed294e5ec0b0fbb35b96c1b3ee6dcdb266dddbe1e935a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.2" + }, + "wheel": { + "hashes": [ + "sha256:78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", + "sha256:e11eefd162658ea59a60a0f6c7d493a7190ea4b9a85e335b33489d9f17e0245e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.36.2" + } + } +} diff --git a/FCRdataLoader/README.md b/FCRdataLoader/README.md new file mode 100644 index 0000000..b10bd50 --- /dev/null +++ b/FCRdataLoader/README.md @@ -0,0 +1,47 @@ +# pytorch dataset for FCR app (currently uses only data from single file) +# How to install dependencies + - make sure you have python3 installed + - if you don't have pipenv already installed run: +```sh +$ pip install pipenv +``` + - now run in directory with `Pipfile` +```sh +$ pipenv install +$ pipenv lock -r > requirements.txt +$ pip install -r requirements.txt +``` + +

+ +# Contents +## FCRdataLoader.fcrdataloader.dataset.FCRtrainDataSet + - containing training data +## FCRdataLoader.fcrdataloader.dataset.FCRtestDataSet + - containing test data +### details about pytorch dataset and DataLoader can be found here: https://pytorch.org/docs/stable/data.html + +

+ +# Example of usage +```Python +from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet +from torch.utils.data import DataLoader + + +train_data = FCRtrainDataSet(50, 5) +loader = DataLoader(dataset=train_data, batch_size=16, shuffle=True) + +model = Network() # some network +criterion = nn.MSELoss() +optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) + +for epoch in range(100): + for (seq, answer) in enumerate(dataloader): + prediction = model(seq) + loss = criterion(prediction, answer) + + optimizer.zero_grad() + loss.backward() + optimizer.step() +``` \ No newline at end of file diff --git a/FCRdataLoader/data/data1.csv b/FCRdataLoader/data/data1.csv new file mode 100644 index 0000000..8e076fc --- /dev/null +++ b/FCRdataLoader/data/data1.csv @@ -0,0 +1,998 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-10-29T14:06:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:07:10.029,0.0,8.092933654785156,0,1,0,1,0,1,0 +2020-10-29T14:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:09:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:10:10.029,0.0,9.235248565673828,0,1,0,1,0,1,0 +2020-10-29T14:11:10.029,10.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:12:10.029,0.0,12.410219192504883,0,1,0,1,0,1,0 +2020-10-29T14:13:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:14:10.029,0.0,8.37138557434082,0,1,0,1,0,1,0 +2020-10-29T14:15:10.029,20.0,0.07390737533569336,0,1,0,1,0,1,0 +2020-10-29T14:16:10.029,6.0,20.59613037109375,0,1,0,1,0,1,0 +2020-10-29T14:17:10.029,0.0,1.4238762855529785,0,1,0,1,0,1,0 +2020-10-29T14:18:10.029,14.0,0.4126596450805664,0,1,0,1,0,1,0 +2020-10-29T14:19:10.029,3.0,20.156076431274414,0,1,0,1,0,1,0 +2020-10-29T14:20:10.029,0.0,3.812490463256836,0,1,0,1,0,1,0 +2020-10-29T14:21:10.029,0.0,1.0089221000671387,0,1,0,1,0,1,0 +2020-10-29T14:22:10.029,0.0,0.984745979309082,0,1,0,1,0,1,0 +2020-10-29T14:23:10.029,0.0,0.3623847961425781,0,1,0,1,0,1,0 +2020-10-29T14:24:10.029,0.0,0.0015397071838378906,0,1,0,1,0,1,0 +2020-10-29T14:25:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:26:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:27:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:28:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:29:10.029,8.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:30:10.029,0.0,8.296137809753418,0,1,0,1,0,1,0 +2020-10-29T14:31:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:32:10.029,0.0,5.666323661804199,0,1,0,1,0,1,0 +2020-10-29T14:33:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:34:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:35:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:36:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:37:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:38:10.029,26.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:39:10.029,0.0,10.709664344787598,0,1,0,1,0,1,0 +2020-10-29T14:40:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:41:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:42:10.029,0.0,5.936244010925293,0,1,0,1,0,1,0 +2020-10-29T14:43:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:44:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:45:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:46:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:47:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:48:10.029,9.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:49:10.029,0.0,7.949969291687012,0,1,0,1,0,1,0 +2020-10-29T14:50:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:51:10.029,342.0,0.0,0,7,0,1,0,1,0 +2020-10-29T14:52:10.029,0.0,21.785860061645508,0,1,0,1,0,1,0 +2020-10-29T14:53:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:54:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:55:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:56:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:57:10.029,9.0,0.0,0,1,0,1,0,1,0 +2020-10-29T14:58:10.029,0.0,6.099068641662598,0,1,0,1,0,1,0 +2020-10-29T14:59:10.029,11.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:00:10.029,5.0,8.87404727935791,0,1,0,1,0,1,0 +2020-10-29T15:01:10.029,7.0,0.14431333541870117,0,1,0,1,0,1,0 +2020-10-29T15:02:10.029,0.0,3.6838326454162598,0,1,0,1,0,1,0 +2020-10-29T15:03:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:04:10.029,5.0,6.429187774658203,0,1,0,1,0,1,0 +2020-10-29T15:05:10.029,0.0,2.1575698852539062,0,1,0,1,0,1,0 +2020-10-29T15:06:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:07:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:09:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:10:10.029,0.0,11.564858436584473,0,1,0,1,0,1,0 +2020-10-29T15:11:10.029,0.0,0.25743913650512695,0,1,0,1,0,1,0 +2020-10-29T15:12:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:13:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:14:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:15:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:16:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:17:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:18:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:19:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:20:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:21:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:22:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:23:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:24:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:25:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:26:10.029,0.0,15.251335144042969,0,1,0,1,0,1,0 +2020-10-29T15:27:10.029,356.0,0.0,0,7,0,1,0,1,0 +2020-10-29T15:28:10.029,0.0,46.44401550292969,0,1,0,1,0,1,0 +2020-10-29T15:29:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:30:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:31:10.029,6.0,1.699239730834961,0,1,0,1,0,1,0 +2020-10-29T15:32:10.029,0.0,1.8481130599975586,0,1,0,1,0,1,0 +2020-10-29T15:33:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:34:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:35:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:36:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:37:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:38:10.029,0.0,2.3151702880859375,0,1,0,1,0,1,0 +2020-10-29T15:39:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:40:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:41:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:42:10.029,15.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:43:10.029,23.0,11.84577751159668,0,1,0,1,0,1,0 +2020-10-29T15:44:10.029,0.0,13.07125473022461,0,1,0,1,0,1,0 +2020-10-29T15:45:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:46:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:47:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:48:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:49:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:50:10.029,0.0,5.202688217163086,0,1,0,1,0,1,0 +2020-10-29T15:51:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:52:10.029,50.0,6.235217094421387,0,1,0,1,0,1,0 +2020-10-29T15:53:10.029,5.0,25.572189331054688,0,1,0,1,0,1,0 +2020-10-29T15:54:10.029,28.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:55:10.029,0.0,19.234643936157227,0,1,0,1,0,1,0 +2020-10-29T15:56:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:57:10.029,0.0,6.527838706970215,0,1,0,1,0,1,0 +2020-10-29T15:58:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T15:59:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:00:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:01:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:02:10.029,13.0,11.12646770477295,0,1,0,1,0,1,0 +2020-10-29T16:03:10.029,8.0,12.76125717163086,0,1,0,1,0,1,0 +2020-10-29T16:04:10.029,0.0,4.127845764160156,0,1,0,1,0,1,0 +2020-10-29T16:05:10.029,0.0,0.1368122100830078,0,1,0,1,0,1,0 +2020-10-29T16:06:10.029,3.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:07:10.029,0.0,4.626254081726074,0,1,0,1,0,1,0 +2020-10-29T16:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:09:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:10:10.029,0.0,6.42384147644043,0,1,0,1,0,1,0 +2020-10-29T16:11:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:12:10.029,6.0,7.250375747680664,0,1,0,1,0,1,0 +2020-10-29T16:13:10.034,6.0,7.434720039367676,0,1,0,1,0,1,0 +2020-10-29T16:14:10.029,0.0,7.118711471557617,0,1,0,1,0,1,0 +2020-10-29T16:15:10.029,0.0,0.36592769622802734,0,1,0,1,0,1,0 +2020-10-29T16:16:10.029,4.5,0.0,0,1,0,1,0,1,0 +2020-10-29T16:17:10.029,5.0,7.875121116638184,0,1,0,1,0,1,0 +2020-10-29T16:18:10.029,6.5,6.107962608337402,0,1,0,1,0,1,0 +2020-10-29T16:19:10.029,0.0,8.611654281616211,0,1,0,1,0,1,0 +2020-10-29T16:20:10.029,0.0,0.5526518821716309,0,1,0,1,0,1,0 +2020-10-29T16:21:10.029,6.0,0.2611522674560547,0,1,0,1,0,1,0 +2020-10-29T16:22:10.029,4.0,11.57531452178955,0,1,0,1,0,1,0 +2020-10-29T16:23:10.029,5.0,3.7174105644226074,0,1,0,1,0,1,0 +2020-10-29T16:24:10.029,0.0,8.499622344970703,0,1,0,1,0,1,0 +2020-10-29T16:25:10.029,0.0,1.018892765045166,0,1,0,1,0,1,0 +2020-10-29T16:26:10.029,7.0,0.36528682708740234,0,1,0,1,0,1,0 +2020-10-29T16:27:10.029,5.0,11.12242317199707,0,1,0,1,0,1,0 +2020-10-29T16:28:10.029,0.0,4.008762359619141,0,1,0,1,0,1,0 +2020-10-29T16:29:10.029,0.0,0.9459342956542969,0,1,0,1,0,1,0 +2020-10-29T16:30:10.029,6.0,0.38293981552124023,0,1,0,1,0,1,0 +2020-10-29T16:31:10.029,0.0,10.566802978515625,0,1,0,1,0,1,0 +2020-10-29T16:32:10.029,0.0,0.2418060302734375,0,1,0,1,0,1,0 +2020-10-29T16:33:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:34:10.029,0.0,8.920586585998535,0,1,0,1,0,1,0 +2020-10-29T16:35:10.029,8.0,0.17075300216674805,0,1,0,1,0,1,0 +2020-10-29T16:36:10.029,0.0,11.595227241516113,0,1,0,1,0,1,0 +2020-10-29T16:37:10.029,0.0,0.06052255630493164,0,1,0,1,0,1,0 +2020-10-29T16:38:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:39:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:40:10.029,0.0,6.924949645996094,0,1,0,1,0,1,0 +2020-10-29T16:41:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:42:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:43:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:44:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:45:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:46:10.029,12.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:47:10.029,9.0,11.044121742248535,0,1,0,1,0,1,0 +2020-10-29T16:48:10.029,7.0,5.830473899841309,0,1,0,1,0,1,0 +2020-10-29T16:49:10.029,0.0,4.056408882141113,0,1,0,1,0,1,0 +2020-10-29T16:50:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:51:10.029,0.0,6.529085159301758,0,1,0,1,0,1,0 +2020-10-29T16:52:10.029,8.0,0.0,0,1,0,1,0,1,0 +2020-10-29T16:53:10.029,5.0,9.057021141052246,0,1,0,1,0,1,0 +2020-10-29T16:54:10.029,0.0,1.7574710845947266,0,1,0,1,0,1,0 +2020-10-29T16:55:10.029,202.0,0.0,0,4,0,1,0,1,0 +2020-10-29T16:56:10.029,52.0,33.17987060546875,0,1,0,1,0,1,0 +2020-10-29T16:57:10.029,0.0,4.32895565032959,0,1,0,1,0,1,0 +2020-10-29T16:58:10.029,0.0,0.5938620567321777,0,1,0,1,0,1,0 +2020-10-29T16:59:10.029,5.0,0.6138205528259277,0,1,0,1,0,1,0 +2020-10-29T17:00:10.029,0.0,5.230597496032715,0,1,0,1,0,1,0 +2020-10-29T17:01:10.029,0.0,0.027581214904785156,0,1,0,1,0,1,0 +2020-10-29T17:02:10.029,8.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:03:10.029,5.0,9.127798080444336,0,1,0,1,0,1,0 +2020-10-29T17:04:10.029,20.0,2.4755353927612305,0,1,0,1,0,1,0 +2020-10-29T17:05:10.029,7.0,20.369178771972656,0,1,0,1,0,1,0 +2020-10-29T17:06:10.029,6.0,3.315825939178467,0,1,0,1,0,1,0 +2020-10-29T17:07:10.029,0.0,4.419093132019043,0,1,0,1,0,1,0 +2020-10-29T17:08:10.029,5.0,0.1558208465576172,0,1,0,1,0,1,0 +2020-10-29T17:09:10.029,0.0,6.724039077758789,0,1,0,1,0,1,0 +2020-10-29T17:10:10.029,0.0,0.4746742248535156,0,1,0,1,0,1,0 +2020-10-29T17:11:10.029,0.0,0.15520572662353516,0,1,0,1,0,1,0 +2020-10-29T17:12:10.029,5.0,0.13993406295776367,0,1,0,1,0,1,0 +2020-10-29T17:13:10.029,5.0,10.159008026123047,0,1,0,1,0,1,0 +2020-10-29T17:14:10.029,0.0,5.231137275695801,0,1,0,1,0,1,0 +2020-10-29T17:15:10.029,0.0,0.714848518371582,0,1,0,1,0,1,0 +2020-10-29T17:16:10.029,4.0,0.1667160987854004,0,1,0,1,0,1,0 +2020-10-29T17:17:10.029,5.0,8.324742317199707,0,1,0,1,0,1,0 +2020-10-29T17:18:10.029,0.0,6.501380920410156,0,1,0,1,0,1,0 +2020-10-29T17:19:10.029,0.0,0.1724538803100586,0,1,0,1,0,1,0 +2020-10-29T17:20:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:21:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:22:10.029,13.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:23:10.029,0.0,13.301803588867188,0,1,0,1,0,1,0 +2020-10-29T17:24:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:25:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:26:10.029,0.0,7.142899513244629,0,1,0,1,0,1,0 +2020-10-29T17:27:10.029,251.0,0.0,0,5,0,1,0,1,0 +2020-10-29T17:28:10.029,0.0,23.88077163696289,0,1,0,1,0,1,0 +2020-10-29T17:29:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:30:10.029,8.5,0.0,0,1,0,1,0,1,0 +2020-10-29T17:31:10.029,4.0,7.784939765930176,0,1,0,1,0,1,0 +2020-10-29T17:32:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:33:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:34:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:35:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:36:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:37:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:38:10.029,6.0,5.221391677856445,0,1,0,1,0,1,0 +2020-10-29T17:39:10.029,0.0,1.9989275932312012,0,1,0,1,0,1,0 +2020-10-29T17:40:10.029,11.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:41:10.029,0.0,10.013413429260254,0,1,0,1,0,1,0 +2020-10-29T17:42:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:43:10.029,180.0,9.235176086425781,0,4,0,1,0,1,0 +2020-10-29T17:44:10.029,0.0,24.981861114501953,0,1,0,1,0,1,0 +2020-10-29T17:45:10.029,317.5,0.0,0,6,0,1,0,1,0 +2020-10-29T17:46:10.029,12.0,74.0953598022461,0,1,0,1,0,1,0 +2020-10-29T17:47:10.029,5.5,0.29892683029174805,0,1,0,1,0,1,0 +2020-10-29T17:48:10.029,5.5,3.1506519317626953,0,1,0,1,0,1,0 +2020-10-29T17:49:10.029,0.0,5.2662763595581055,0,1,0,1,0,1,0 +2020-10-29T17:50:10.029,0.0,1.3602228164672852,0,1,0,1,0,1,0 +2020-10-29T17:51:10.029,0.0,0.6114144325256348,0,1,0,1,0,1,0 +2020-10-29T17:52:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:53:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:54:10.029,0.0,1.7974481582641602,0,1,0,1,0,1,0 +2020-10-29T17:55:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:56:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:57:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:58:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T17:59:10.029,0.0,6.526606559753418,0,1,0,1,0,1,0 +2020-10-29T18:00:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:01:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:02:10.029,0.0,5.383490562438965,0,1,0,1,0,1,0 +2020-10-29T18:03:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:04:10.029,8.0,8.05236530303955,0,1,0,1,0,1,0 +2020-10-29T18:05:10.029,0.0,8.077051162719727,0,1,0,1,0,1,0 +2020-10-29T18:06:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:07:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:09:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:10:10.032,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:11:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:12:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:13:10.029,12.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:14:10.029,0.0,11.618339538574219,0,1,0,1,0,1,0 +2020-10-29T18:15:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:16:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:17:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:18:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:19:10.029,4.0,5.411664009094238,0,1,0,1,0,1,0 +2020-10-29T18:20:10.029,0.0,2.1512250900268555,0,1,0,1,0,1,0 +2020-10-29T18:21:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:22:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:23:10.029,162.0,0.0,0,3,0,1,0,1,0 +2020-10-29T18:24:10.029,4.0,17.396459579467773,0,1,0,1,0,1,0 +2020-10-29T18:25:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:26:10.029,7.0,2.2944464683532715,0,1,0,1,0,1,0 +2020-10-29T18:27:10.029,0.0,4.356158256530762,0,1,0,1,0,1,0 +2020-10-29T18:28:10.029,16.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:29:10.029,0.0,14.886369705200195,0,1,0,1,0,1,0 +2020-10-29T18:30:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:31:10.029,0.0,5.8511962890625,0,1,0,1,0,1,0 +2020-10-29T18:32:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:33:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:34:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:35:10.029,0.0,3.7542366981506348,0,1,0,1,0,1,0 +2020-10-29T18:36:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:37:10.029,252.0,0.0,0,5,0,1,0,1,0 +2020-10-29T18:38:10.029,5.0,29.539676666259766,0,1,0,1,0,1,0 +2020-10-29T18:39:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:40:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:41:10.029,0.0,5.665234565734863,0,1,0,1,0,1,0 +2020-10-29T18:42:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:43:10.029,5.5,0.0,0,1,0,1,0,1,0 +2020-10-29T18:44:10.029,5.0,4.7825517654418945,0,1,0,1,0,1,0 +2020-10-29T18:45:10.029,0.0,1.490556240081787,0,1,0,1,0,1,0 +2020-10-29T18:46:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:47:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:48:10.029,6.0,4.517246246337891,0,1,0,1,0,1,0 +2020-10-29T18:49:10.029,0.0,3.4358997344970703,0,1,0,1,0,1,0 +2020-10-29T18:50:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:51:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:52:10.029,3.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:53:10.029,7.0,6.487791061401367,0,1,0,1,0,1,0 +2020-10-29T18:54:10.031,0.0,8.03091812133789,0,1,0,1,0,1,0 +2020-10-29T18:55:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T18:56:10.029,0.0,9.796905517578125,0,1,0,1,0,1,0 +2020-10-29T18:57:10.029,7.0,0.17009544372558594,0,1,0,1,0,1,0 +2020-10-29T18:58:10.029,4.0,10.307432174682617,0,1,0,1,0,1,0 +2020-10-29T18:59:10.029,0.0,2.21535062789917,0,1,0,1,0,1,0 +2020-10-29T19:00:10.029,0.0,0.5963382720947266,0,1,0,1,0,1,0 +2020-10-29T19:01:10.029,8.0,0.15051031112670898,0,1,0,1,0,1,0 +2020-10-29T19:02:10.029,9.0,11.291885375976562,0,1,0,1,0,1,0 +2020-10-29T19:03:10.029,5.0,9.199846267700195,0,1,0,1,0,1,0 +2020-10-29T19:04:10.029,0.0,3.387078285217285,0,1,0,1,0,1,0 +2020-10-29T19:05:10.029,6.0,1.3799810409545898,0,1,0,1,0,1,0 +2020-10-29T19:06:10.029,0.0,14.252178192138672,0,1,0,1,0,1,0 +2020-10-29T19:07:10.029,0.0,1.2471351623535156,0,1,0,1,0,1,0 +2020-10-29T19:08:10.029,6.0,0.6201796531677246,0,1,0,1,0,1,0 +2020-10-29T19:09:10.029,38.0,11.787551879882812,0,1,0,1,0,1,0 +2020-10-29T19:10:10.029,0.0,29.54815673828125,0,1,0,1,0,1,0 +2020-10-29T19:11:10.029,5.0,0.4530959129333496,0,1,0,1,0,1,0 +2020-10-29T19:12:10.029,0.0,9.565673828125,0,1,0,1,0,1,0 +2020-10-29T19:13:10.029,0.0,0.3695106506347656,0,1,0,1,0,1,0 +2020-10-29T19:14:10.029,0.0,0.36685609817504883,0,1,0,1,0,1,0 +2020-10-29T19:15:10.029,239.0,0.0,0,5,0,1,0,1,0 +2020-10-29T19:16:10.029,0.0,39.20317459106445,0,1,0,1,0,1,0 +2020-10-29T19:17:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:18:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:19:10.029,0.0,3.2760848999023438,0,1,0,1,0,1,0 +2020-10-29T19:20:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:21:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:22:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:23:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:24:10.029,8.0,3.594817638397217,0,1,0,1,0,1,0 +2020-10-29T19:25:10.029,0.0,3.721907615661621,0,1,0,1,0,1,0 +2020-10-29T19:26:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:27:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:28:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:29:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:30:10.029,5.0,4.781319618225098,0,1,0,1,0,1,0 +2020-10-29T19:31:10.029,0.0,4.42032527923584,0,1,0,1,0,1,0 +2020-10-29T19:32:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:33:10.029,0.0,6.181086540222168,0,1,0,1,0,1,0 +2020-10-29T19:34:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:35:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:36:10.029,0.0,8.427678108215332,0,1,0,1,0,1,0 +2020-10-29T19:37:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:38:10.029,8.0,9.44045352935791,0,1,0,1,0,1,0 +2020-10-29T19:39:10.029,0.0,7.80142879486084,0,1,0,1,0,1,0 +2020-10-29T19:40:10.029,0.0,0.18083858489990234,0,1,0,1,0,1,0 +2020-10-29T19:41:10.029,0.0,0.17366552352905273,0,1,0,1,0,1,0 +2020-10-29T19:42:10.029,0.0,0.16237449645996094,0,1,0,1,0,1,0 +2020-10-29T19:43:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:44:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:45:10.029,3.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:46:10.029,0.0,4.790645599365234,0,1,0,1,0,1,0 +2020-10-29T19:47:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:48:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:49:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:50:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:51:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:52:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:53:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:54:10.029,0.0,6.535590171813965,0,1,0,1,0,1,0 +2020-10-29T19:55:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:56:10.029,0.0,6.322650909423828,0,1,0,1,0,1,0 +2020-10-29T19:57:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:58:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T19:59:10.029,5.0,6.71759033203125,0,1,0,1,0,1,0 +2020-10-29T20:00:10.029,0.0,4.385467529296875,0,1,0,1,0,1,0 +2020-10-29T20:01:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:02:10.029,0.0,8.942854881286621,0,1,0,1,0,1,0 +2020-10-29T20:03:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:04:10.029,0.0,7.598843574523926,0,1,0,1,0,1,0 +2020-10-29T20:05:10.029,5.0,0.44580507278442383,0,1,0,1,0,1,0 +2020-10-29T20:06:10.029,7.0,10.363755226135254,0,1,0,1,0,1,0 +2020-10-29T20:07:10.029,0.0,9.921998023986816,0,1,0,1,0,1,0 +2020-10-29T20:08:10.029,6.0,1.1291399002075195,0,1,0,1,0,1,0 +2020-10-29T20:09:10.029,4.0,11.877375602722168,0,1,0,1,0,1,0 +2020-10-29T20:10:10.029,0.0,4.260271072387695,0,1,0,1,0,1,0 +2020-10-29T20:11:10.029,0.0,2.4076004028320312,0,1,0,1,0,1,0 +2020-10-29T20:12:10.029,0.0,1.1791324615478516,0,1,0,1,0,1,0 +2020-10-29T20:13:10.029,6.0,0.48076391220092773,0,1,0,1,0,1,0 +2020-10-29T20:14:10.029,6.0,10.632257461547852,0,1,0,1,0,1,0 +2020-10-29T20:15:10.029,0.0,7.27094841003418,0,1,0,1,0,1,0 +2020-10-29T20:16:10.029,0.0,0.6268486976623535,0,1,0,1,0,1,0 +2020-10-29T20:17:10.029,0.0,0.19138622283935547,0,1,0,1,0,1,0 +2020-10-29T20:18:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:19:10.029,0.0,7.540426254272461,0,1,0,1,0,1,0 +2020-10-29T20:20:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:21:10.029,4.5,0.0,0,1,0,1,0,1,0 +2020-10-29T20:22:10.029,0.0,6.261873245239258,0,1,0,1,0,1,0 +2020-10-29T20:23:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:24:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:25:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:26:10.029,0.0,7.58897590637207,0,1,0,1,0,1,0 +2020-10-29T20:27:10.029,40.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:28:10.029,0.0,14.05528450012207,0,1,0,1,0,1,0 +2020-10-29T20:29:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:30:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:31:10.029,15.0,8.196240425109863,0,1,0,1,0,1,0 +2020-10-29T20:32:10.029,0.0,11.093750953674316,0,1,0,1,0,1,0 +2020-10-29T20:33:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:34:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:35:10.029,0.0,7.550601959228516,0,1,0,1,0,1,0 +2020-10-29T20:36:10.029,8.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:37:10.029,0.0,12.223976135253906,0,1,0,1,0,1,0 +2020-10-29T20:38:10.029,0.0,0.39418792724609375,0,1,0,1,0,1,0 +2020-10-29T20:39:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:40:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:41:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:42:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:43:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:44:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:45:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:46:10.029,93.0,0.0,0,2,0,1,0,1,0 +2020-10-29T20:47:10.029,45.0,15.535181045532227,0,1,0,1,0,1,0 +2020-10-29T20:48:10.029,0.0,8.485981941223145,0,1,0,1,0,1,0 +2020-10-29T20:49:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:50:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:51:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:52:10.029,6.0,2.0895204544067383,0,1,0,1,0,1,0 +2020-10-29T20:53:10.029,0.0,1.9933748245239258,0,1,0,1,0,1,0 +2020-10-29T20:54:10.029,13.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:55:10.029,0.0,10.309151649475098,0,1,0,1,0,1,0 +2020-10-29T20:56:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:57:10.029,0.0,4.175436973571777,0,1,0,1,0,1,0 +2020-10-29T20:58:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T20:59:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:00:10.029,22.0,4.75355339050293,0,1,0,1,0,1,0 +2020-10-29T21:01:10.029,0.0,19.050439834594727,0,1,0,1,0,1,0 +2020-10-29T21:02:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:03:10.029,0.0,8.154370307922363,0,1,0,1,0,1,0 +2020-10-29T21:04:10.029,0.0,0.8469753265380859,0,1,0,1,0,1,0 +2020-10-29T21:05:10.029,0.0,0.34672975540161133,0,1,0,1,0,1,0 +2020-10-29T21:06:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:07:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:09:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:10:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:11:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:12:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:13:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:14:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:15:10.029,350.0,4.703037261962891,0,7,0,1,0,1,0 +2020-10-29T21:16:10.029,0.0,40.69681167602539,0,1,0,1,0,1,0 +2020-10-29T21:17:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:18:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:19:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:20:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:21:10.029,0.0,3.2666473388671875,0,1,0,1,0,1,0 +2020-10-29T21:22:10.030,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:23:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:24:10.029,0.0,3.8338260650634766,0,1,0,1,0,1,0 +2020-10-29T21:25:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:26:10.029,0.0,3.327702522277832,0,1,0,1,0,1,0 +2020-10-29T21:27:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:28:10.029,0.0,4.763402938842773,0,1,0,1,0,1,0 +2020-10-29T21:29:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:30:10.029,0.0,6.9983673095703125,0,1,0,1,0,1,0 +2020-10-29T21:31:10.029,0.0,0.13197612762451172,0,1,0,1,0,1,0 +2020-10-29T21:32:10.029,0.0,0.1350231170654297,0,1,0,1,0,1,0 +2020-10-29T21:33:10.029,9.0,0.11713886260986328,0,1,0,1,0,1,0 +2020-10-29T21:34:10.029,0.0,11.736916542053223,0,1,0,1,0,1,0 +2020-10-29T21:35:10.029,0.0,0.12194442749023438,0,1,0,1,0,1,0 +2020-10-29T21:36:10.029,0.0,0.11377096176147461,0,1,0,1,0,1,0 +2020-10-29T21:37:10.029,96.0,0.0,0,2,0,1,0,1,0 +2020-10-29T21:38:10.029,0.0,17.7043399810791,0,1,0,1,0,1,0 +2020-10-29T21:39:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:40:10.029,0.0,6.96132755279541,0,1,0,1,0,1,0 +2020-10-29T21:41:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:42:10.029,5.0,7.154741287231445,0,1,0,1,0,1,0 +2020-10-29T21:43:10.029,4.0,1.5402956008911133,0,1,0,1,0,1,0 +2020-10-29T21:44:10.029,0.0,1.4707465171813965,0,1,0,1,0,1,0 +2020-10-29T21:45:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:46:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:47:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:48:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:49:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:50:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:51:10.029,8.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:52:10.029,5.0,8.71021556854248,0,1,0,1,0,1,0 +2020-10-29T21:53:10.029,0.0,1.9077439308166504,0,1,0,1,0,1,0 +2020-10-29T21:54:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:55:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:56:10.029,22.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:57:10.029,0.0,9.41596508026123,0,1,0,1,0,1,0 +2020-10-29T21:58:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T21:59:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:00:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:01:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:02:10.029,0.0,5.084737777709961,0,1,0,1,0,1,0 +2020-10-29T22:03:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:04:10.029,0.0,7.03974723815918,0,1,0,1,0,1,0 +2020-10-29T22:05:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:06:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:07:10.029,0.0,6.234905242919922,0,1,0,1,0,1,0 +2020-10-29T22:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:09:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:10:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:11:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:12:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:13:10.029,15.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:14:10.029,0.0,10.901710510253906,0,1,0,1,0,1,0 +2020-10-29T22:15:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:16:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:17:10.029,0.0,5.1751298904418945,0,1,0,1,0,1,0 +2020-10-29T22:18:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:19:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:20:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:21:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:22:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:23:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:24:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:25:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:26:10.029,6.0,5.5879058837890625,0,1,0,1,0,1,0 +2020-10-29T22:27:10.029,0.0,4.2069902420043945,0,1,0,1,0,1,0 +2020-10-29T22:28:10.029,191.0,0.0,0,4,0,1,0,1,0 +2020-10-29T22:29:10.029,4.0,17.969911575317383,0,1,0,1,0,1,0 +2020-10-29T22:30:10.029,11.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:31:10.029,6.0,6.570542335510254,0,1,0,1,0,1,0 +2020-10-29T22:32:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:33:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:34:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:35:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:36:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:37:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:38:10.029,0.0,3.4944934844970703,0,1,0,1,0,1,0 +2020-10-29T22:39:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:40:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:41:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:42:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:43:10.029,0.0,4.713992118835449,0,1,0,1,0,1,0 +2020-10-29T22:44:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:45:10.029,0.0,6.209532737731934,0,1,0,1,0,1,0 +2020-10-29T22:46:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:47:10.029,3.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:48:10.029,0.0,4.264294624328613,0,1,0,1,0,1,0 +2020-10-29T22:49:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:50:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:51:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:52:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:53:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:54:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:55:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:56:10.030,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:57:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:58:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T22:59:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:00:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:01:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:02:10.029,287.0,16.195114135742188,0,6,0,1,0,1,0 +2020-10-29T23:03:10.029,0.0,38.41609191894531,0,1,0,1,0,1,0 +2020-10-29T23:04:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:05:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:06:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:07:10.029,0.0,2.750178813934326,0,1,0,1,0,1,0 +2020-10-29T23:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:09:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:10:10.029,0.0,2.201333522796631,0,1,0,1,0,1,0 +2020-10-29T23:11:10.029,12.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:12:10.029,0.0,8.798405647277832,0,1,0,1,0,1,0 +2020-10-29T23:13:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:14:10.029,59.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:15:10.029,0.0,22.150562286376953,0,1,0,1,0,1,0 +2020-10-29T23:16:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:17:10.029,0.0,6.523756980895996,0,1,0,1,0,1,0 +2020-10-29T23:18:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:19:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:20:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:21:10.029,0.0,6.732582092285156,0,1,0,1,0,1,0 +2020-10-29T23:22:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:23:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:24:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:25:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:26:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:27:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:28:10.029,6.0,4.772862434387207,0,1,0,1,0,1,0 +2020-10-29T23:29:10.029,0.0,5.109376907348633,0,1,0,1,0,1,0 +2020-10-29T23:30:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:31:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:32:10.029,0.0,4.709239959716797,0,1,0,1,0,1,0 +2020-10-29T23:33:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:34:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:35:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:36:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:37:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:38:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:39:10.029,0.0,4.721630096435547,0,1,0,1,0,1,0 +2020-10-29T23:40:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:41:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:42:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:43:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:44:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:45:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:46:10.029,0.0,4.526134490966797,0,1,0,1,0,1,0 +2020-10-29T23:47:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:48:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:49:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:50:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:51:10.029,0.0,6.65598201751709,0,1,0,1,0,1,0 +2020-10-29T23:52:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:53:10.029,10.5,4.829201698303223,0,1,0,1,0,1,0 +2020-10-29T23:54:10.029,0.0,8.031954765319824,0,1,0,1,0,1,0 +2020-10-29T23:55:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:56:10.029,0.0,6.480889320373535,0,1,0,1,0,1,0 +2020-10-29T23:57:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-29T23:58:10.029,0.0,8.18777084350586,0,1,0,1,0,1,0 +2020-10-29T23:59:10.029,5.0,0.05682659149169922,0,1,0,1,0,1,0 +2020-10-30T00:00:10.029,0.0,8.405426025390625,0,1,0,1,0,1,0 +2020-10-30T00:01:10.029,0.0,0.46718788146972656,0,1,0,1,0,1,0 +2020-10-30T00:02:10.029,0.0,0.4610404968261719,0,1,0,1,0,1,0 +2020-10-30T00:03:10.029,0.0,0.44185352325439453,0,1,0,1,0,1,0 +2020-10-30T00:04:10.029,16.0,0.04636049270629883,0,1,0,1,0,1,0 +2020-10-30T00:05:10.029,0.0,16.4781436920166,0,1,0,1,0,1,0 +2020-10-30T00:06:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:07:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:08:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:09:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:10:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:11:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:12:10.029,146.0,0.0,0,3,0,1,0,1,0 +2020-10-30T00:13:10.029,0.0,11.152569770812988,0,1,0,1,0,1,0 +2020-10-30T00:14:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:15:10.029,4.0,1.4813618659973145,0,1,0,1,0,1,0 +2020-10-30T00:16:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:17:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:18:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:19:10.029,0.0,5.509651184082031,0,1,0,1,0,1,0 +2020-10-30T00:20:10.029,8.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:21:10.029,0.0,7.602544784545898,0,1,0,1,0,1,0 +2020-10-30T00:22:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:23:10.029,5.5,0.0,0,1,0,1,0,1,0 +2020-10-30T00:24:10.029,48.0,5.560975074768066,0,1,0,1,0,1,0 +2020-10-30T00:25:10.029,0.0,25.713239669799805,0,1,0,1,0,1,0 +2020-10-30T00:26:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:27:10.029,103.0,0.0,0,2,0,1,0,1,0 +2020-10-30T00:28:10.029,5.0,37.42110061645508,0,1,0,1,0,1,0 +2020-10-30T00:29:10.029,0.0,0.2669506072998047,0,1,0,1,0,1,0 +2020-10-30T00:30:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:31:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:32:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:33:10.029,11.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:34:10.029,0.0,10.127628326416016,0,1,0,1,0,1,0 +2020-10-30T00:35:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:36:10.029,6.0,2.062410831451416,0,1,0,1,0,1,0 +2020-10-30T00:37:10.029,0.0,2.933445453643799,0,1,0,1,0,1,0 +2020-10-30T00:38:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:39:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:40:10.029,6.0,5.1230573654174805,0,1,0,1,0,1,0 +2020-10-30T00:41:10.029,5.0,4.622580528259277,0,1,0,1,0,1,0 +2020-10-30T00:42:10.029,0.0,3.64579439163208,0,1,0,1,0,1,0 +2020-10-30T00:43:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:44:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:45:10.029,0.0,8.640241622924805,0,1,0,1,0,1,0 +2020-10-30T00:46:10.029,88.0,0.6256647109985352,0,2,0,1,0,1,0 +2020-10-30T00:47:10.029,0.0,23.388309478759766,0,1,0,1,0,1,0 +2020-10-30T00:48:10.029,0.0,0.029178142547607422,0,1,0,1,0,1,0 +2020-10-30T00:49:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:50:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:51:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:52:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:53:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:54:10.029,0.0,6.692010879516602,0,1,0,1,0,1,0 +2020-10-30T00:55:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:56:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:57:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:58:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T00:59:10.029,6.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:00:10.029,4.0,4.232800483703613,0,1,0,1,0,1,0 +2020-10-30T01:01:10.029,0.0,1.832387924194336,0,1,0,1,0,1,0 +2020-10-30T01:02:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:03:10.029,0.0,4.666852951049805,0,1,0,1,0,1,0 +2020-10-30T01:04:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:05:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:06:10.029,16.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:07:10.029,0.0,11.525970458984375,0,1,0,1,0,1,0 +2020-10-30T01:08:10.029,5.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:09:10.029,0.0,6.69814395904541,0,1,0,1,0,1,0 +2020-10-30T01:10:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:11:10.029,7.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:12:10.029,0.0,9.639561653137207,0,1,0,1,0,1,0 +2020-10-30T01:13:10.029,6.0,0.04102945327758789,0,1,0,1,0,1,0 +2020-10-30T01:14:10.029,0.0,8.68164348602295,0,1,0,1,0,1,0 +2020-10-30T01:15:10.029,0.0,0.03974151611328125,0,1,0,1,0,1,0 +2020-10-30T01:16:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:17:10.029,4.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:18:10.029,0.0,7.123937606811523,0,1,0,1,0,1,0 +2020-10-30T01:19:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:20:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:21:10.029,0.0,0.0,0,1,0,1,0,1,0 +2020-10-30T01:22:10.029,193.0,0.0,1,4,0,1,0,1,0 +2020-10-30T01:23:10.029,0.0,20.622085571289062,1,1,0,1,0,1,0 +2020-10-30T01:24:10.029,67.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:25:10.029,0.0,22.343050003051758,1,1,0,1,0,1,0 +2020-10-30T01:26:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:27:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:28:10.029,0.0,1.5565133094787598,1,1,0,1,0,1,0 +2020-10-30T01:29:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:30:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:31:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:32:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:33:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:34:10.029,16.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:35:10.029,0.0,10.564291954040527,1,1,0,1,0,1,0 +2020-10-30T01:36:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:37:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:38:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:39:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:40:10.029,0.0,4.639544486999512,1,1,0,1,0,1,0 +2020-10-30T01:41:10.029,12.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:42:10.029,3.0,9.352090835571289,1,1,0,1,0,1,0 +2020-10-30T01:43:10.029,103.0,0.0,1,2,0,1,0,1,0 +2020-10-30T01:44:10.029,25.0,18.862722396850586,1,1,0,1,0,1,0 +2020-10-30T01:45:10.029,15.0,0.4382319450378418,1,1,0,1,0,1,0 +2020-10-30T01:46:10.029,9.0,3.2147507667541504,1,1,0,1,0,1,0 +2020-10-30T01:47:10.029,4.0,3.5339250564575195,1,1,0,1,0,1,0 +2020-10-30T01:48:10.029,12.0,2.22397518157959,1,1,0,1,0,1,0 +2020-10-30T01:49:10.029,6.0,12.766632080078125,1,1,0,1,0,1,0 +2020-10-30T01:50:10.029,22.0,4.93465518951416,1,1,0,1,0,1,0 +2020-10-30T01:51:10.029,7.0,30.895244598388672,1,1,0,1,0,1,0 +2020-10-30T01:52:10.029,0.0,13.804502487182617,1,1,0,1,0,1,0 +2020-10-30T01:53:10.029,0.0,6.213910102844238,1,1,0,1,0,1,0 +2020-10-30T01:54:10.029,0.0,3.929741382598877,1,1,0,1,0,1,0 +2020-10-30T01:55:10.029,0.0,1.8266377449035645,1,1,0,1,0,1,0 +2020-10-30T01:56:10.029,0.0,0.7386393547058105,1,1,0,1,0,1,0 +2020-10-30T01:57:10.029,0.0,0.08167219161987305,1,1,0,1,0,1,0 +2020-10-30T01:58:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T01:59:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:00:10.029,56.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:01:10.029,0.0,23.045989990234375,1,1,0,1,0,1,0 +2020-10-30T02:02:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:03:10.029,7.0,4.46674919128418,1,1,0,1,0,1,0 +2020-10-30T02:04:10.029,0.0,3.792720317840576,1,1,0,1,0,1,0 +2020-10-30T02:05:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:06:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:07:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:08:10.029,7.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:09:10.029,6.0,6.497780799865723,1,1,0,1,0,1,0 +2020-10-30T02:10:10.029,0.0,1.8255853652954102,1,1,0,1,0,1,0 +2020-10-30T02:11:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:12:10.029,13.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:13:10.029,5.0,12.702003479003906,1,1,0,1,0,1,0 +2020-10-30T02:14:10.029,0.0,1.0951943397521973,1,1,0,1,0,1,0 +2020-10-30T02:15:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:16:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:17:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:18:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:19:10.029,6.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:20:10.029,0.0,6.788580894470215,1,1,0,1,0,1,0 +2020-10-30T02:21:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:22:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:23:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:24:10.029,6.0,5.22747802734375,1,1,0,1,0,1,0 +2020-10-30T02:25:10.029,0.0,4.602972030639648,1,1,0,1,0,1,0 +2020-10-30T02:26:10.029,7.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:27:10.029,0.0,7.530411720275879,1,1,0,1,0,1,0 +2020-10-30T02:28:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:29:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:30:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:31:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:32:10.029,0.0,6.599978446960449,1,1,0,1,0,1,0 +2020-10-30T02:33:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:34:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:35:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:36:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:37:10.029,7.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:38:10.029,6.0,7.402435302734375,1,1,0,1,0,1,0 +2020-10-30T02:39:10.029,0.0,3.9300241470336914,1,1,0,1,0,1,0 +2020-10-30T02:40:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:41:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:42:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:43:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:44:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:45:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:46:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:47:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:48:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:49:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:50:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:51:10.029,148.0,0.0,1,3,0,1,0,1,0 +2020-10-30T02:52:10.029,4.0,47.786895751953125,1,1,0,1,0,1,0 +2020-10-30T02:53:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:54:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:55:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:56:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:57:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:58:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T02:59:10.029,7.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:00:10.029,0.0,4.276817321777344,1,1,0,1,0,1,0 +2020-10-30T03:01:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:02:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:03:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:04:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:05:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:06:10.029,7.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:07:10.029,0.0,5.683505058288574,1,1,0,1,0,1,0 +2020-10-30T03:08:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:09:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:10:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:11:10.029,6.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:12:10.029,0.0,6.073542594909668,1,1,0,1,0,1,0 +2020-10-30T03:13:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:14:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:15:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:16:10.029,6.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:17:10.029,0.0,6.23763370513916,1,1,0,1,0,1,0 +2020-10-30T03:18:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:19:10.029,21.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:20:10.029,0.0,9.620964050292969,1,1,0,1,0,1,0 +2020-10-30T03:21:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:22:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:23:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:24:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:25:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:26:10.029,4.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:27:10.029,0.0,4.076115608215332,1,1,0,1,0,1,0 +2020-10-30T03:28:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:29:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:30:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:31:10.029,0.0,4.418728828430176,1,1,0,1,0,1,0 +2020-10-30T03:32:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:33:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:34:10.029,7.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:35:10.029,0.0,7.036018371582031,1,1,0,1,0,1,0 +2020-10-30T03:36:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:37:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:38:10.029,131.0,0.0,1,3,0,1,0,1,0 +2020-10-30T03:39:10.029,198.0,18.23261260986328,1,4,0,1,0,1,0 +2020-10-30T03:40:10.029,0.0,25.32823944091797,1,1,0,1,0,1,0 +2020-10-30T03:41:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:42:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:43:10.029,4.0,1.189723014831543,1,1,0,1,0,1,0 +2020-10-30T03:44:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:45:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:46:10.029,0.0,1.6236763000488281,1,1,0,1,0,1,0 +2020-10-30T03:47:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:48:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:49:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:50:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:51:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:52:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:53:10.029,5.0,5.261150360107422,1,1,0,1,0,1,0 +2020-10-30T03:54:10.029,0.0,4.008538246154785,1,1,0,1,0,1,0 +2020-10-30T03:55:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:56:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:57:10.029,0.0,5.861323356628418,1,1,0,1,0,1,0 +2020-10-30T03:58:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T03:59:10.029,4.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:00:10.029,5.0,4.629955291748047,1,1,0,1,0,1,0 +2020-10-30T04:01:10.029,170.0,4.353062629699707,1,3,0,1,0,1,0 +2020-10-30T04:02:10.029,0.0,17.637008666992188,1,1,0,1,0,1,0 +2020-10-30T04:03:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:04:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:05:10.029,6.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:06:10.029,9.0,5.1784515380859375,1,1,0,1,0,1,0 +2020-10-30T04:07:10.029,6.0,4.843062400817871,1,1,0,1,0,1,0 +2020-10-30T04:08:10.029,5.0,1.1457085609436035,1,1,0,1,0,1,0 +2020-10-30T04:09:10.029,0.0,2.015923500061035,1,1,0,1,0,1,0 +2020-10-30T04:10:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:11:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:12:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:13:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:14:10.029,4.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:15:10.029,6.0,4.304697036743164,1,1,0,1,0,1,0 +2020-10-30T04:16:10.029,6.0,5.193272590637207,1,1,0,1,0,1,0 +2020-10-30T04:17:10.029,0.0,4.814790725708008,1,1,0,1,0,1,0 +2020-10-30T04:18:10.029,5.0,0.04473114013671875,1,1,0,1,0,1,0 +2020-10-30T04:19:10.029,0.0,8.80262565612793,1,1,0,1,0,1,0 +2020-10-30T04:20:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:21:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:22:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:23:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:24:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:25:10.029,5.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:26:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:27:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:28:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:29:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:30:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:31:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:32:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:33:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:34:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:35:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:36:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:37:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:38:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:39:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:40:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:41:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:42:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:43:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:44:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:45:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:46:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:47:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:48:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:49:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:50:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:51:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:52:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:53:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 +2020-10-30T04:54:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 +2020-10-30T04:55:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 +2020-10-30T04:56:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 +2020-10-30T04:57:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:58:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T04:59:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T05:00:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T05:01:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T05:02:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T05:03:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T05:04:10.029,0.0,0.0,1,1,0,1,0,1,0 +2020-10-30T05:05:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:06:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:07:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:08:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:09:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:10:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:11:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:12:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:13:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:14:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:15:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:16:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:17:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:18:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:19:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:20:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:21:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:22:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:23:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:24:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:25:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:26:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:27:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:28:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:29:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:30:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:31:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:32:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:33:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:34:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:35:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:36:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:37:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:38:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:39:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:40:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:41:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:42:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:43:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:44:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:45:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:46:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:47:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:48:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:49:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:50:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:51:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:52:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:53:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:54:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:55:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:56:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:57:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:58:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T05:59:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:00:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:01:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:02:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:03:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:04:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:05:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:06:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:07:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:08:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:09:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:10:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:11:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:12:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:13:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:14:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:15:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:16:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:17:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:18:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:19:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:20:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:21:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:22:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:23:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:24:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:25:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:26:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:27:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:28:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:29:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:30:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:31:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:32:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:33:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:34:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:35:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:36:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:37:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:38:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:39:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:40:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:41:10.029,0.0,0.0,-1,1,0,1,0,1,0 +2020-10-30T06:42:10.029,0.0,0.0,-1,1,0,1,0,1,0 diff --git a/FCRdataLoader/data/data2.csv b/FCRdataLoader/data/data2.csv new file mode 100644 index 0000000..aa2ee65 --- /dev/null +++ b/FCRdataLoader/data/data2.csv @@ -0,0 +1,218 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-11-27T12:44:35.519,6.0,9.435382843017578,0,1,0,1,0,1,0 +2020-11-27T12:45:35.519,17.0,8.48339557647705,0,1,0,1,0,1,0 +2020-11-27T12:46:35.519,10.0,23.755578994750977,0,1,0,1,0,1,0 +2020-11-27T12:47:35.519,5.0,11.223828315734863,0,1,0,1,0,1,0 +2020-11-27T12:48:35.519,43.0,7.788514137268066,0,1,0,1,0,1,0 +2020-11-27T12:49:35.519,10.0,44.337806701660156,0,1,0,1,0,1,0 +2020-11-27T12:50:35.519,4.0,8.92234992980957,0,1,0,1,0,1,0 +2020-11-27T12:51:35.519,5.0,7.031393051147461,0,1,0,1,0,1,0 +2020-11-27T12:52:35.519,6.0,7.168320655822754,0,1,0,1,0,1,0 +2020-11-27T12:53:35.519,8.0,7.513263702392578,0,1,0,1,0,1,0 +2020-11-27T12:54:35.519,4.0,9.204913139343262,0,1,0,1,0,1,0 +2020-11-27T12:55:35.519,14.0,6.85783052444458,0,1,0,1,0,1,0 +2020-11-27T12:56:35.519,17.33333333333333,20.1556453704834,0,1,0,1,0,1,0 +2020-11-27T12:57:35.519,12.0,15.963403701782227,0,1,0,1,0,1,0 +2020-11-27T12:58:35.519,4.0,12.1525297164917,0,1,0,1,0,1,0 +2020-11-27T12:59:35.519,4.0,6.540548324584961,0,1,0,1,0,1,0 +2020-11-27T13:00:35.519,6.0,5.795986652374268,0,1,0,1,0,1,0 +2020-11-27T13:01:35.519,7.0,6.7676005363464355,0,1,0,1,0,1,0 +2020-11-27T13:02:35.519,13.0,8.009368896484375,0,1,0,1,0,1,0 +2020-11-27T13:03:35.519,6.0,16.15487289428711,0,1,0,1,0,1,0 +2020-11-27T13:04:35.519,12.0,7.246679306030273,0,1,0,1,0,1,0 +2020-11-27T13:05:35.519,6.0,16.62491798400879,0,1,0,1,0,1,0 +2020-11-27T13:06:35.519,5.0,7.308323860168457,0,1,0,1,0,1,0 +2020-11-27T13:07:35.519,31.5,6.282835483551025,0,1,0,1,0,1,0 +2020-11-27T13:08:35.519,4.0,19.173612594604492,0,1,0,1,0,1,0 +2020-11-27T13:09:35.519,8.0,6.749720573425293,0,1,0,1,0,1,0 +2020-11-27T13:10:35.519,15.0,9.082990646362305,0,1,0,1,0,1,0 +2020-11-27T13:11:35.519,6.0,18.638320922851562,0,1,0,1,0,1,0 +2020-11-27T13:12:35.519,5.0,8.155741691589355,0,1,0,1,0,1,0 +2020-11-27T13:13:35.520,11.0,7.225281715393066,0,1,0,1,0,1,0 +2020-11-27T13:14:35.519,15.0,13.923819541931152,0,1,0,1,0,1,0 +2020-11-27T13:15:35.519,5.0,17.63955307006836,0,1,0,1,0,1,0 +2020-11-27T13:16:35.519,4.0,7.650225639343262,0,1,0,1,0,1,0 +2020-11-27T13:17:35.519,0.0,6.992892742156982,0,1,0,1,0,1,0 +2020-11-27T13:18:35.519,4.0,4.934108257293701,0,1,0,1,0,1,0 +2020-11-27T13:19:35.519,7.0,4.724740505218506,0,1,0,1,0,1,0 +2020-11-27T13:20:35.519,15.0,7.635990142822266,0,1,0,1,0,1,0 +2020-11-27T13:21:35.519,160.0,9.775514602661133,0,3,0,1,0,1,0 +2020-11-27T13:22:35.519,50.0,63.986534118652344,0,1,0,1,0,1,0 +2020-11-27T13:23:35.519,7.0,11.747467994689941,0,1,0,1,0,1,0 +2020-11-27T13:24:35.519,6.0,9.03959846496582,0,1,0,1,0,1,0 +2020-11-27T13:25:35.519,96.0,7.851625442504883,0,2,0,1,0,1,0 +2020-11-27T13:26:35.519,4.0,39.70072555541992,0,1,0,1,0,1,0 +2020-11-27T13:27:35.519,4.0,9.647521018981934,0,1,0,1,0,1,0 +2020-11-27T13:28:35.519,5.0,9.178874015808105,0,1,0,1,0,1,0 +2020-11-27T13:29:35.519,6.0,10.066906929016113,0,1,0,1,0,1,0 +2020-11-27T13:30:35.519,81.0,10.560870170593262,0,2,0,1,0,1,0 +2020-11-27T13:31:35.519,988.0,104.04701232910156,0,20,0,1,0,1,0 +2020-11-27T13:32:35.519,6.0,380.4679870605469,0,1,0,1,0,1,0 +2020-11-27T13:33:35.519,6.0,14.513042449951172,0,1,0,1,0,1,0 +2020-11-27T13:34:35.519,5.0,10.953410148620605,0,1,0,1,0,1,0 +2020-11-27T13:35:35.519,14.0,9.55089282989502,0,1,0,1,0,1,0 +2020-11-27T13:36:35.519,5.0,12.438122749328613,0,1,0,1,0,1,0 +2020-11-27T13:37:35.519,6.0,7.341762542724609,0,1,0,1,0,1,0 +2020-11-27T13:38:35.519,6.0,6.660945892333984,0,1,0,1,0,1,0 +2020-11-27T13:39:35.519,6.0,6.816884517669678,0,1,0,1,0,1,0 +2020-11-27T13:40:35.519,7.0,6.6493377685546875,0,1,0,1,0,1,0 +2020-11-27T13:41:35.519,8.0,5.92299222946167,0,1,0,1,0,1,0 +2020-11-27T13:42:35.519,6.0,5.889991283416748,0,1,0,1,0,1,0 +2020-11-27T13:43:35.519,5.0,6.110908508300781,0,1,0,1,0,1,0 +2020-11-27T13:44:35.519,4.0,5.786072254180908,0,1,0,1,0,1,0 +2020-11-27T13:45:35.519,0.0,5.522675514221191,0,1,0,1,0,1,0 +2020-11-27T13:46:35.519,8.0,4.02607536315918,0,1,0,1,0,1,0 +2020-11-27T13:47:35.519,6.0,11.456418991088867,0,1,0,1,0,1,0 +2020-11-27T13:48:35.519,0.0,5.000259876251221,0,1,0,1,0,1,0 +2020-11-27T13:49:35.519,43.0,3.127312660217285,0,1,0,1,0,1,0 +2020-11-27T13:50:35.519,0.0,30.338163375854492,0,1,0,1,0,1,0 +2020-11-27T13:51:35.521,5.0,2.966379165649414,0,1,0,1,0,1,0 +2020-11-27T13:52:35.519,5.0,3.754279613494873,0,1,0,1,0,1,0 +2020-11-27T13:53:35.519,0.0,3.5321502685546875,0,1,0,1,0,1,0 +2020-11-27T13:54:35.519,6.0,2.0308921337127686,0,1,0,1,0,1,0 +2020-11-27T13:55:35.519,6.0,4.462838172912598,0,1,0,1,0,1,0 +2020-11-27T13:56:35.519,0.0,4.041638374328613,0,1,0,1,0,1,0 +2020-11-27T13:57:35.519,7.0,2.098271369934082,0,1,0,1,0,1,0 +2020-11-27T13:58:35.519,105.5,5.880876064300537,0,2,0,1,0,1,0 +2020-11-27T13:59:35.519,0.0,110.12590026855469,0,1,0,1,0,1,0 +2020-11-27T14:00:35.519,287.0,2.413583278656006,0,6,0,1,0,1,0 +2020-11-27T14:01:35.519,14.0,335.0638732910156,0,1,0,1,0,1,0 +2020-11-27T14:02:35.519,5.0,5.429197311401367,0,1,0,1,0,1,0 +2020-11-27T14:03:35.519,8.0,4.946113109588623,0,1,0,1,0,1,0 +2020-11-27T14:04:35.519,5.0,6.548556327819824,0,1,0,1,0,1,0 +2020-11-27T14:05:35.519,6.0,5.965693950653076,0,1,0,1,0,1,0 +2020-11-27T14:06:35.519,726.0,6.134999752044678,0,14,0,1,0,1,0 +2020-11-27T14:07:35.519,4665.0,1021.2295532226562,0,47,0,1,0,1,0 +2020-11-27T14:08:35.519,0.0,3070.738037109375,0,1,0,1,0,1,0 +2020-11-27T14:09:35.520,9.0,14.447136878967285,0,1,0,1,0,1,0 +2020-11-27T14:10:35.519,5.0,10.972953796386719,0,1,0,1,0,1,0 +2020-11-27T14:11:35.519,4.0,7.169851303100586,0,1,0,1,0,1,0 +2020-11-27T14:12:35.519,7.0,5.326818943023682,0,1,0,1,0,1,0 +2020-11-27T14:13:35.519,6.0,5.767587661743164,0,1,0,1,0,1,0 +2020-11-27T14:14:35.519,6.0,5.402779579162598,0,1,0,1,0,1,0 +2020-11-27T14:15:35.519,6.0,5.32569694519043,0,1,0,1,0,1,0 +2020-11-27T14:16:35.519,18.0,4.896035671234131,0,1,0,1,0,1,0 +2020-11-27T14:17:35.519,3209.0,6.206113815307617,0,47,0,1,0,1,0 +2020-11-27T14:18:35.519,5.0,591.1841430664062,0,1,0,1,0,1,0 +2020-11-27T14:19:35.519,4.0,5.264420986175537,0,1,0,1,0,1,0 +2020-11-27T14:20:35.519,4.0,3.9137449264526367,0,1,0,1,0,1,0 +2020-11-27T14:21:35.519,6.0,3.76383638381958,0,1,0,1,0,1,0 +2020-11-27T14:22:35.519,6.0,4.28965950012207,0,1,0,1,0,1,0 +2020-11-27T14:23:35.519,5.0,4.339838027954102,0,1,0,1,0,1,0 +2020-11-27T14:24:35.519,15.5,4.126605033874512,0,1,0,1,0,1,0 +2020-11-27T14:25:35.519,16.0,6.732624530792236,0,1,0,1,0,1,0 +2020-11-27T14:26:35.519,7.0,7.165763854980469,0,1,0,1,0,1,0 +2020-11-27T14:27:35.519,5.0,5.099540710449219,0,1,0,1,0,1,0 +2020-11-27T14:28:35.519,4.0,4.222059726715088,0,1,0,1,0,1,0 +2020-11-27T14:29:35.519,4.0,4.934439659118652,0,1,0,1,0,1,0 +2020-11-27T14:30:35.519,5.0,4.902071475982666,0,1,0,1,0,1,0 +2020-11-27T14:31:35.519,7.0,5.144791603088379,0,1,0,1,0,1,0 +2020-11-27T14:32:35.519,26.0,6.1177077293396,0,1,0,1,0,1,0 +2020-11-27T14:33:35.519,2638.5,28.131595611572266,0,47,0,1,0,1,0 +2020-11-27T14:34:35.519,1067.0,616.666748046875,0,21,0,1,0,1,0 +2020-11-27T14:35:35.519,21.0,101.96915435791016,0,1,0,1,0,1,0 +2020-11-27T14:36:35.519,5.0,8.676255226135254,0,1,0,1,0,1,0 +2020-11-27T14:37:35.519,115.0,6.9384965896606445,0,2,0,1,0,1,0 +2020-11-27T14:38:35.519,5.0,57.02287292480469,0,1,0,1,0,1,0 +2020-11-27T14:39:35.519,7.0,11.969968795776367,0,1,0,1,0,1,0 +2020-11-27T14:40:35.519,6.0,12.453564643859863,0,1,0,1,0,1,0 +2020-11-27T14:41:35.519,7.0,12.684252738952637,0,1,0,1,0,1,0 +2020-11-27T14:42:35.519,7.0,12.762392044067383,0,1,0,1,0,1,0 +2020-11-27T14:43:35.519,6.0,10.544450759887695,0,1,0,1,0,1,0 +2020-11-27T14:44:35.519,16.0,7.521728515625,0,1,0,1,0,1,0 +2020-11-27T14:45:35.519,5.0,9.791584014892578,0,1,0,1,0,1,0 +2020-11-27T14:46:35.519,8.0,6.312222480773926,0,1,0,1,0,1,0 +2020-11-27T14:47:35.519,4.0,6.674623966217041,0,1,0,1,0,1,0 +2020-11-27T14:48:35.519,5.0,5.500234127044678,0,1,0,1,0,1,0 +2020-11-27T14:49:35.519,7.0,5.7142205238342285,0,1,0,1,0,1,0 +2020-11-27T14:50:35.519,5.0,6.574368953704834,0,1,0,1,0,1,0 +2020-11-27T14:51:35.519,4.0,5.641943454742432,0,1,0,1,0,1,0 +2020-11-27T14:52:35.519,48.0,5.242447853088379,0,1,0,1,0,1,0 +2020-11-27T14:53:35.519,10.0,31.18132781982422,0,1,0,1,0,1,0 +2020-11-27T14:54:35.519,17.0,6.470431327819824,0,1,0,1,0,1,0 +2020-11-27T14:55:35.519,92.0,17.268766403198242,0,2,0,1,0,1,0 +2020-11-27T14:56:35.519,5.0,94.94540405273438,0,1,0,1,0,1,0 +2020-11-27T14:57:35.519,0.0,6.798692226409912,0,1,0,1,0,1,0 +2020-11-27T14:58:35.519,5.0,5.002079963684082,0,1,0,1,0,1,0 +2020-11-27T14:59:35.519,5.0,5.192873477935791,0,1,0,1,0,1,0 +2020-11-27T15:00:35.519,47.0,5.093393325805664,0,1,0,1,0,1,0 +2020-11-27T15:01:35.519,6.0,66.24467468261719,0,1,0,1,0,1,0 +2020-11-27T15:02:35.519,106.5,6.989488124847412,0,2,0,1,0,1,0 +2020-11-27T15:03:35.519,18.0,114.1700668334961,0,1,0,1,0,1,0 +2020-11-27T15:04:35.519,4.0,9.336390495300293,0,1,0,1,0,1,0 +2020-11-27T15:05:35.519,5.0,6.51580810546875,0,1,0,1,0,1,0 +2020-11-27T15:06:35.519,8.0,5.369061470031738,0,1,0,1,0,1,0 +2020-11-27T15:07:35.519,7.0,5.92508602142334,0,1,0,1,0,1,0 +2020-11-27T15:08:35.519,6.0,6.801459312438965,0,1,0,1,0,1,0 +2020-11-27T15:09:35.519,5.0,7.037836074829102,0,1,0,1,0,1,0 +2020-11-27T15:10:35.519,6.5,6.7276225090026855,0,1,0,1,0,1,0 +2020-11-27T15:11:35.519,6.0,6.155050754547119,0,1,0,1,0,1,0 +2020-11-27T15:12:35.519,65.0,5.615631103515625,0,1,0,1,0,1,0 +2020-11-27T15:13:35.519,5.0,59.99169921875,0,1,0,1,0,1,0 +2020-11-27T15:14:35.519,0.0,5.508193492889404,0,1,0,1,0,1,0 +2020-11-27T15:15:35.519,7.0,3.9370083808898926,1,1,0,1,0,1,0 +2020-11-27T15:16:35.519,6.0,5.107851505279541,1,1,0,1,0,1,0 +2020-11-27T15:17:35.519,0.0,4.678167343139648,1,1,0,1,0,1,0 +2020-11-27T15:18:35.519,6.0,2.993084669113159,1,1,0,1,0,1,0 +2020-11-27T15:19:35.519,0.0,4.081082344055176,1,1,0,1,0,1,0 +2020-11-27T15:20:35.519,0.0,2.3085193634033203,1,1,0,1,0,1,0 +2020-11-27T15:21:35.519,0.0,1.4350025653839111,1,1,0,1,0,1,0 +2020-11-27T15:22:35.519,0.0,0.8783214092254639,1,1,0,1,0,1,0 +2020-11-27T15:23:35.519,0.0,0.5170314311981201,1,1,0,1,0,1,0 +2020-11-27T15:24:35.519,0.0,0.30115222930908203,1,1,0,1,0,1,0 +2020-11-27T15:25:35.519,0.0,0.19430434703826904,1,1,0,1,0,1,0 +2020-11-27T15:26:35.519,0.0,0.06121242046356201,1,1,0,1,0,1,0 +2020-11-27T15:27:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:28:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:29:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:30:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:31:35.520,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:32:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:33:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:34:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:35:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:36:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:37:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:38:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:39:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:40:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:41:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:42:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:43:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:44:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:45:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:46:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:47:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:48:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:49:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:50:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:51:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:52:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:53:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:54:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:55:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:56:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:57:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:58:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T15:59:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:00:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:01:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:02:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:03:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:04:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:05:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:06:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:07:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:08:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:09:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:10:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:11:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:12:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:13:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:14:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:15:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:16:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:17:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:18:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:19:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:20:35.519,0.0,0.0,-1,1,0,1,0,1,0 diff --git a/FCRdataLoader/data/data4.csv b/FCRdataLoader/data/data4.csv new file mode 100644 index 0000000..0d25180 --- /dev/null +++ b/FCRdataLoader/data/data4.csv @@ -0,0 +1,116 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-12-15T11:49:00,0.0,2.880878210067749,0,1,0,1,0,1,0 +2020-12-15T11:50:00,0.0,1.698030710220337,0,1,0,1,0,1,0 +2020-12-15T11:51:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T11:52:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T11:53:00,0.0,0.3707929849624634,0,1,0,1,0,1,0 +2020-12-15T11:54:00,0.0,0.45026612281799316,0,1,0,1,0,1,0 +2020-12-15T11:55:00,0.0,0.41281116008758545,0,1,0,1,0,1,0 +2020-12-15T11:56:00,0.0,0.3778296709060669,0,1,0,1,0,1,0 +2020-12-15T11:57:00,0.0,0.42775797843933105,0,1,0,1,0,1,0 +2020-12-15T11:58:00,0.0,52.15424346923828,0,1,0,1,0,1,0 +2020-12-15T11:59:00,0.0,1.3606882095336914,0,1,0,1,0,1,0 +2020-12-15T12:00:00,0.0,1.7871627807617188,0,1,0,1,0,1,0 +2020-12-15T12:01:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:02:00,0.0,0.29342174530029297,0,1,0,1,0,1,0 +2020-12-15T12:03:00,0.0,0.22346079349517822,0,1,0,1,0,1,0 +2020-12-15T12:04:00,0.0,0.20592844486236572,0,1,0,1,0,1,0 +2020-12-15T12:05:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:06:00,0.0,0.2854079008102417,0,1,0,1,0,1,0 +2020-12-15T12:07:00,0.0,2.0263261795043945,0,1,0,1,0,1,0 +2020-12-15T12:08:00,0.0,1.193394660949707,0,1,0,1,0,1,0 +2020-12-15T12:09:00,0.0,0.6382132768630981,0,1,0,1,0,1,0 +2020-12-15T12:10:00,0.0,14.562797546386719,0,1,0,1,0,1,0 +2020-12-15T12:11:00,0.0,0.7512111663818359,0,1,0,1,0,1,0 +2020-12-15T12:12:00,0.0,0.8914639949798584,0,1,0,1,0,1,0 +2020-12-15T12:13:00,0.0,0.7688208818435669,0,1,0,1,0,1,0 +2020-12-15T12:14:00,0.0,0.7653945684432983,0,1,0,1,0,1,0 +2020-12-15T12:15:00,0.0,1.1794123649597168,0,1,0,1,0,1,0 +2020-12-15T12:16:00,0.0,1.1221184730529785,0,1,0,1,0,1,0 +2020-12-15T12:17:00,0.0,1.2044415473937988,0,1,0,1,0,1,0 +2020-12-15T12:18:00,0.0,1.167661190032959,0,1,0,1,0,1,0 +2020-12-15T12:19:00,0.0,29.40463638305664,0,1,0,1,0,1,0 +2020-12-15T12:20:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:21:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:22:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:23:00,0.0,0.26250433921813965,0,1,0,1,0,1,0 +2020-12-15T12:24:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:25:00,0.0,0.01861274242401123,0,1,0,1,0,1,0 +2020-12-15T12:26:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:27:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:28:00,0.0,1.031090259552002,0,1,0,1,0,1,0 +2020-12-15T12:29:00,0.0,0.1976233720779419,0,1,0,1,0,1,0 +2020-12-15T12:30:00,0.0,0.16222071647644043,0,1,0,1,0,1,0 +2020-12-15T12:31:00,0.0,0.45725953578948975,0,1,0,1,0,1,0 +2020-12-15T12:32:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:33:00,0.0,0.6110225915908813,0,1,0,1,0,1,0 +2020-12-15T12:34:00,0.0,0.5869172811508179,0,1,0,1,0,1,0 +2020-12-15T12:35:00,0.0,824.6790161132812,0,1,0,1,0,1,0 +2020-12-15T12:36:00,0.0,143.26446533203125,0,1,0,1,0,1,0 +2020-12-15T12:37:00,0.0,0.9533004760742188,0,1,0,1,0,1,0 +2020-12-15T12:38:00,0.0,6.044524192810059,0,1,0,1,0,1,0 +2020-12-15T12:39:00,0.0,2.429046630859375,0,1,0,1,0,1,0 +2020-12-15T12:40:00,0.0,2.7120466232299805,0,1,0,1,0,1,0 +2020-12-15T12:41:00,0.0,2.1224734783172607,0,1,0,1,0,1,0 +2020-12-15T12:42:00,0.0,1.8528714179992676,0,1,0,1,0,1,0 +2020-12-15T12:43:00,0.0,1.8902008533477783,0,1,0,1,0,1,0 +2020-12-15T12:44:00,0.0,1.9152851104736328,0,1,0,1,0,1,0 +2020-12-15T12:45:00,0.0,1.8422601222991943,0,1,0,1,0,1,0 +2020-12-15T12:46:00,0.0,32.213531494140625,0,1,0,1,0,1,0 +2020-12-15T12:47:00,0.0,4.091504096984863,0,1,0,1,0,1,0 +2020-12-15T12:48:00,0.0,10.508217811584473,0,1,0,1,0,1,0 +2020-12-15T12:49:00,0.0,2.708798885345459,0,1,0,1,0,1,0 +2020-12-15T12:50:00,0.0,2.3304004669189453,0,1,0,1,0,1,0 +2020-12-15T12:51:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:52:00,0.0,1.0770142078399658,0,1,0,1,0,1,0 +2020-12-15T12:53:00,0.0,0.8451101779937744,0,1,0,1,0,1,0 +2020-12-15T12:54:00,0.0,1.1430389881134033,0,1,0,1,0,1,0 +2020-12-15T12:55:00,0.0,1.2068374156951904,0,1,0,1,0,1,0 +2020-12-15T12:56:00,0.0,728.1805419921875,0,1,0,1,0,1,0 +2020-12-15T12:57:00,0.0,15.266721725463867,0,1,0,1,0,1,0 +2020-12-15T12:58:00,0.0,2.8867130279541016,0,1,0,1,0,1,0 +2020-12-15T12:59:00,0.0,2.095475196838379,0,1,0,1,0,1,0 +2020-12-15T13:00:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:01:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:02:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:03:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:04:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:05:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:06:00,0.0,204.33526611328125,1,1,0,1,0,1,0 +2020-12-15T13:07:00,0.0,2.2900924682617188,1,1,0,1,0,1,0 +2020-12-15T13:08:00,0.0,4.229484558105469,1,1,0,1,0,1,0 +2020-12-15T13:09:00,0.0,2.979954957962036,1,1,0,1,0,1,0 +2020-12-15T13:10:00,0.0,0.8080476522445679,1,1,0,1,0,1,0 +2020-12-15T13:11:00,0.0,2.4772040843963623,1,1,0,1,0,1,0 +2020-12-15T13:12:00,0.0,2.2220327854156494,1,1,0,1,0,1,0 +2020-12-15T13:13:00,0.0,365.6095275878906,1,1,0,1,0,1,0 +2020-12-15T13:14:00,0.0,2.6103553771972656,1,1,0,1,0,1,0 +2020-12-15T13:15:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:16:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:17:00,0.0,730.0831909179688,1,1,0,1,0,1,0 +2020-12-15T13:18:00,0.0,5.1552734375,1,1,0,1,0,1,0 +2020-12-15T13:19:00,0.0,5.315293312072754,1,1,0,1,0,1,0 +2020-12-15T13:20:00,0.0,3.747960090637207,1,1,0,1,0,1,0 +2020-12-15T13:21:00,0.0,3.213507652282715,1,1,0,1,0,1,0 +2020-12-15T13:22:00,0.0,3.209949493408203,1,1,0,1,0,1,0 +2020-12-15T13:23:00,0.0,12.009992599487305,1,1,0,1,0,1,0 +2020-12-15T13:24:00,0.0,68.0241928100586,1,1,0,1,0,1,0 +2020-12-15T13:25:00,0.0,5.560791492462158,1,1,0,1,0,1,0 +2020-12-15T13:26:00,0.0,4.588584899902344,1,1,0,1,0,1,0 +2020-12-15T13:27:00,0.0,2.873000144958496,1,1,0,1,0,1,0 +2020-12-15T13:28:00,0.0,2.874795436859131,1,1,0,1,0,1,0 +2020-12-15T13:29:00,0.0,2.6174628734588623,1,1,0,1,0,1,0 +2020-12-15T13:30:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:31:00,0.0,0.7121289968490601,-1,1,0,1,0,1,0 +2020-12-15T13:32:00,0.0,0.5510646104812622,-1,1,0,1,0,1,0 +2020-12-15T13:33:00,0.0,0.3965524435043335,-1,1,0,1,0,1,0 +2020-12-15T13:34:00,0.0,0.26016998291015625,-1,1,0,1,0,1,0 +2020-12-15T13:35:00,0.0,0.129294753074646,-1,1,0,1,0,1,0 +2020-12-15T13:36:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:37:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:38:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:39:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:40:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:41:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:42:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:43:00,0.0,0.0,-1,1,0,1,0,1,0 diff --git a/FCRdataLoader/fcrdataloader/__init__.py b/FCRdataLoader/fcrdataloader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FCRdataLoader/fcrdataloader/data.py b/FCRdataLoader/fcrdataloader/data.py new file mode 100644 index 0000000..0a6d194 --- /dev/null +++ b/FCRdataLoader/fcrdataloader/data.py @@ -0,0 +1,5 @@ +from pathlib import Path + + +__DATA_DIR = Path(__file__).absolute().parents[1].joinpath('data') +DATA_FILES = sorted(__DATA_DIR.glob("**/*.csv")) diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py new file mode 100644 index 0000000..c3eb6a8 --- /dev/null +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -0,0 +1,116 @@ +from torch.utils.data import Dataset +from pathlib import Path +from .data import DATA_FILES +from typing import List +from abc import ABC, abstractmethod +import numpy as np +import torch + + +MAX_TEST_S = 1000 +TEST_TRAIN_RATIO = 0.3 + + +def test_size(file_sizes: List[int]) -> int: + """ + given list of rows in datafiles returns test size + """ + + total = sum(file_sizes) + test_s = min(MAX_TEST_S, int(TEST_TRAIN_RATIO * total)) + return test_s + + +def override(f): + """ + function to indicate that method is overriden + from class Dataset + """ + + return f + + +class FCRdataSet(Dataset, ABC): + """ + abstract class representing FCR dataset + Currently only uses data from one file + """ + + @abstractmethod + def load_series(self, file_path: Path) -> np.array: + """ + must return np.array of shape (n, 9) + with dtype: float32 + """ + pass + + def __init__(self, seq_len: int, pred_step: int): + """ + seq_len: is number of following records retuned as a sequence + pred_step: is distance between last record in sequence and correctly predicted row + """ + + self.series = self.load_series(DATA_FILES[0]) + # now we have tensor of shape (n, 9) + self.series = torch.from_numpy(self.series) + self.seq_len = seq_len + self.pred_step = pred_step + self.size = self.series.shape[0] - seq_len - pred_step + 1 + + if self.size <= 0: + raise ValueError(f"Error with given seq_len: {seq_len} and pred_step: {pred_step}" + "dataset can't return any data (they are probably too big)") + + if seq_len <= 0 or pred_step <= 0: + raise ValueError(f"seq_len and pred_step can't be negative") + + @override + def __getitem__(self, idx) -> torch.tensor: + """ + returns tuple (seq, pred) where: + seq is torch.tensor of shape (seq_len, 9) + pred is torch.tensor of shape (9,) + """ + + return (self.series[idx:idx + self.seq_len], + self.series[idx + self.seq_len + self.pred_step - 1]) + + @override + def __len__(self): + return self.size + + +class FCRtrainDataSet(FCRdataSet): + """ + represents FCR dataset for training + """ + + @override + def load_series(self, file_path: Path): + series = np.genfromtxt( + file_path, + delimiter=',', + skip_header=1, + usecols=range(1, 10), + dtype=np.float32 + ) + test_s = test_size([series.shape[0]]) + return series[:-test_s] + + +class FCRtestDataSet(FCRdataSet): + """ + represents FCR dataset for testing + """ + + @override + def load_series(self, file_path: Path): + series = np.genfromtxt( + file_path, + delimiter=',', + skip_header=1, + usecols=range(1, 10), + dtype=np.float32 + ) + test_s = test_size([series.shape[0]]) + return series[-test_s:] diff --git a/FCRdataLoader/setup.py b/FCRdataLoader/setup.py new file mode 100644 index 0000000..23a3360 --- /dev/null +++ b/FCRdataLoader/setup.py @@ -0,0 +1,156 @@ +"""A setuptools based setup module. +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +Modified by Madoshakalaka@Github (dependency links added) +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +from os import path + +# io.open is needed for projects that support Python 2.7 +# It ensures open() defaults to text mode with universal newlines, +# and accepts an argument to specify the text encoding +# Python 3 only projects can skip this import +from io import open + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, "README.md"), encoding="utf-8") as f: + long_description = f.read() + +# Arguments marked as "Required" below must be included for upload to PyPI. +# Fields marked as "Optional" may be commented out. + +setup( + # This is the name of your project. The first time you publish this + # package, this name will be registered for you. It will determine how + # users can install this project, e.g.: + # + # $ pip install sampleproject + # + # And where it will live on PyPI: https://pypi.org/project/sampleproject/ + # + # There are some restrictions on what makes a valid project name + # specification here: + # https://packaging.python.org/specifications/core-metadata/#name + name="FCRdataLoader", # Required + # Versions should comply with PEP 440: + # https://www.python.org/dev/peps/pep-0440/ + # + # For a discussion on single-sourcing the version across setup.py and the + # project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version="0.0.0", # Required + # This is a one-line description or tagline of what your project does. This + # corresponds to the "Summary" metadata field: + # https://packaging.python.org/specifications/core-metadata/#summary + # This is an optional longer description of your project that represents + # the body of text which users will see when they visit PyPI. + # + # Often, this is the same as your README, so you can just read it in from + # that file directly (as we have already done above) + # + # This field corresponds to the "Description" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-optional + # Denotes that our long_description is in Markdown; valid values are + # text/plain, text/x-rst, and text/markdown + # + # Optional if long_description is written in reStructuredText (rst) but + # required for plain-text or Markdown; if unspecified, "applications should + # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and + # fall back to text/plain if it is not valid rst" (see link below) + # + # This field corresponds to the "Description-Content-Type" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional + # This should be a valid link to your project's main homepage. + # + # This field corresponds to the "Home-Page" metadata field: + # https://packaging.python.org/specifications/core-metadata/#home-page-optional + # This should be your name or the name of the organization which owns the + # project. + # This should be a valid email address corresponding to the author listed + # above. + # Classifiers help users find your project by categorizing it. + # + # For a list of valid classifiers, see https://pypi.org/classifiers/ + + # This field adds keywords for your project which will appear on the + # project page. What does your project relate to? + # + # Note that this is a string of words separated by whitespace, not a list. + # You can just specify package directories manually here if your project is + # simple. Or you can use find_packages(). + # + # Alternatively, if you just want to distribute a single Python file, use + # the `py_modules` argument instead as follows, which will expect a file + # called `my_module.py` to exist: + # + # py_modules=["my_module"], + # + packages=find_packages(exclude=["contrib", "docs", "tests"]), # Required + # Specify which Python versions you support. In contrast to the + # 'Programming Language' classifiers above, 'pip install' will check this + # and refuse to install the project if the version does not match. If you + # do not support Python 2, you can simplify this to '>=3.5' or similar, see + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4", + # This field lists other packages that your project depends on to run. + # Any package you put here will be installed by pip when your project is + # installed, so they must be valid existing projects. + # + # For an analysis of "install_requires" vs pip's requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + "numpy==1.19.5", + "torch==1.7.1", + "typing-extensions==3.7.4.3", + ], # Optional + # List additional groups of dependencies here (e.g. development + # dependencies). Users will be able to install these using the "extras" + # syntax, for example: + # + # $ pip install sampleproject[dev] + # + # Similar to `install_requires` above, these must be valid existing + # projects. + # If there are data files included in your packages that need to be + # installed, specify them here. + # + # Sometimes you’ll want to use packages that are properly arranged with + # setuptools, but are not published to PyPI. In those cases, you can specify + # a list of one or more dependency_links URLs where the package can + # be downloaded, along with some additional hints, and setuptools + # will find and install the package correctly. + # see https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi + # + dependency_links=[], + # If using Python 2.6 or earlier, then these have to be included in + # MANIFEST.in as well. + # package_data={"sample": ["package_data.dat"]}, # Optional + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. See: + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files + # + # In this case, 'data_file' will be installed into '/my_data' + # data_files=[("my_data", ["data/data_file"])], # Optional + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # `pip` to create the appropriate form of executable for the target + # platform. + # + # For example, the following would provide a command called `sample` which + # executes the function `main` from this package when invoked: + # entry_points={"console_scripts": ["sample=sample:main"]}, # Optional + # List additional URLs that are relevant to your project as a dict. + # + # This field corresponds to the "Project-URL" metadata fields: + # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use + # + # Examples listed include a pattern for specifying where the package tracks + # issues, where the source is hosted, where to say thanks to the package + # maintainers, and where to support the project financially. The key is + # what's used to render the link text on PyPI. +) -- GitLab From 5821ffa50df3a6b5aeccc49e69447b11b13f74f0 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Wed, 6 Jan 2021 14:44:18 +0100 Subject: [PATCH 08/26] small comment change --- FCRdataLoader/fcrdataloader/dataset.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py index c3eb6a8..b9831d9 100644 --- a/FCRdataLoader/fcrdataloader/dataset.py +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -13,7 +13,7 @@ TEST_TRAIN_RATIO = 0.3 def test_size(file_sizes: List[int]) -> int: """ - given list of rows in datafiles returns test size + given list of sizes of datasets in datafiles returns test size """ total = sum(file_sizes) @@ -23,8 +23,7 @@ def test_size(file_sizes: List[int]) -> int: def override(f): """ - function to indicate that method is overriden - from class Dataset + function to indicate that method is overridden """ return f -- GitLab From 25d8c79339868d757dbd6b0472c1abc245a99c3b Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Thu, 7 Jan 2021 21:30:17 +0100 Subject: [PATCH 09/26] critical bugfix --- FCRdataLoader/fcrdataloader/dataset.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py index b9831d9..58581b2 100644 --- a/FCRdataLoader/fcrdataloader/dataset.py +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -49,30 +49,35 @@ class FCRdataSet(Dataset, ABC): pred_step: is distance between last record in sequence and correctly predicted row """ - self.series = self.load_series(DATA_FILES[0]) + series = self.load_series(DATA_FILES[0]) + self.x, self.y = np.hsplit(series, [2]) # split (n,9) to (n,2), (n,7) # now we have tensor of shape (n, 9) - self.series = torch.from_numpy(self.series) + self.x = torch.from_numpy(self.x) + self.y = torch.from_numpy(self.y) self.seq_len = seq_len self.pred_step = pred_step - self.size = self.series.shape[0] - seq_len - pred_step + 1 + self.size = self.x.shape[0] - seq_len - pred_step + 1 if self.size <= 0: raise ValueError(f"Error with given seq_len: {seq_len} and pred_step: {pred_step}" "dataset can't return any data (they are probably too big)") - if seq_len <= 0 or pred_step <= 0: - raise ValueError(f"seq_len and pred_step can't be negative") + if seq_len <= 0: + raise ValueError(f"seq_len must") + + if pred_step < 0: + raise ValueError(f"pred_step can't be negative") @override def __getitem__(self, idx) -> torch.tensor: """ returns tuple (seq, pred) where: - seq is torch.tensor of shape (seq_len, 9) - pred is torch.tensor of shape (9,) + seq is torch.tensor of shape (seq_len, 2) + pred is torch.tensor of shape (7,) """ - return (self.series[idx:idx + self.seq_len], - self.series[idx + self.seq_len + self.pred_step - 1]) + return (self.x[idx:idx + self.seq_len], + self.y[idx + self.seq_len + self.pred_step - 1]) @override def __len__(self): -- GitLab From 19bd626de1e323ecafddc2601c5dfb1c82cbc3eb Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Thu, 7 Jan 2021 22:59:45 +0100 Subject: [PATCH 10/26] same issue, finaly resolved. --- FCRdataLoader/fcrdataloader/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py index 58581b2..0738a99 100644 --- a/FCRdataLoader/fcrdataloader/dataset.py +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -50,7 +50,7 @@ class FCRdataSet(Dataset, ABC): """ series = self.load_series(DATA_FILES[0]) - self.x, self.y = np.hsplit(series, [2]) # split (n,9) to (n,2), (n,7) + self.x, self.y = np.hsplit(series, [3]) # split (n,9) to (n,3), (n,6) # now we have tensor of shape (n, 9) self.x = torch.from_numpy(self.x) self.y = torch.from_numpy(self.y) @@ -72,8 +72,8 @@ class FCRdataSet(Dataset, ABC): def __getitem__(self, idx) -> torch.tensor: """ returns tuple (seq, pred) where: - seq is torch.tensor of shape (seq_len, 2) - pred is torch.tensor of shape (7,) + seq is torch.tensor of shape (seq_len, 3) + pred is torch.tensor of shape (6,) """ return (self.x[idx:idx + self.seq_len], -- GitLab From 1efa7fcf6157f9857620e5dce97b9f63e3323bca Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Thu, 14 Jan 2021 21:12:57 +0100 Subject: [PATCH 11/26] remove unused import --- FCRdataLoader/fcrdataloader/dataset.py | 104 +++++++++++++++++-------- 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py index 0738a99..9e8f528 100644 --- a/FCRdataLoader/fcrdataloader/dataset.py +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -1,4 +1,4 @@ -from torch.utils.data import Dataset +from torch.utils.data import Dataset, WeightedRandomSampler from pathlib import Path from .data import DATA_FILES from typing import List @@ -8,7 +8,7 @@ import torch MAX_TEST_S = 1000 -TEST_TRAIN_RATIO = 0.3 +TEST_TRAIN_RATIO = 0.4 def test_size(file_sizes: List[int]) -> int: @@ -21,14 +21,6 @@ def test_size(file_sizes: List[int]) -> int: return test_s -def override(f): - """ - function to indicate that method is overridden - """ - - return f - - class FCRdataSet(Dataset, ABC): """ abstract class representing FCR dataset @@ -36,39 +28,82 @@ class FCRdataSet(Dataset, ABC): """ @abstractmethod - def load_series(self, file_path: Path) -> np.array: + def load_series(self, file_path: Path) -> (np.array, np.array): """ - must return np.array of shape (n, 9) - with dtype: float32 + must return tuple (x, y) where: + LEGEND: + t - timestamp + v(t) - avg rsp time at time t + p(t) - avg rsp time prediction at time t + split(t) - split value corresponding to prediction at time t + solv(t) - solution to cp-problem at time t (obtained from v(t)) + TUPLE: + x is np.array od dtype: float32 with values (v(t), p(t), split(t)), shape (n, 3) + y is np.array od dtype: float32 with values (solv(t)), shape (n, 6) """ pass - def __init__(self, seq_len: int, pred_step: int): + def __init__(self, seq_len: int, pred_step: int, transforms=None): """ seq_len: is number of following records retuned as a sequence pred_step: is distance between last record in sequence and correctly predicted row + transforms: transformation applited input data. + if returned data is (x, y) then with transormation its (transorms(x), y) """ - series = self.load_series(DATA_FILES[0]) - self.x, self.y = np.hsplit(series, [3]) # split (n,9) to (n,3), (n,6) - # now we have tensor of shape (n, 9) - self.x = torch.from_numpy(self.x) - self.y = torch.from_numpy(self.y) + if seq_len < 0: + raise ValueError(f"seq_len can't be negative") + + if pred_step < 0: + raise ValueError(f"pred_step can't be negative") + + x, y = self.load_series(DATA_FILES[1]) # for now only one file + x_cols = np.hsplit(x, x.shape[1]) + + x_cols = self.__push_cols(x_cols, range(1, 3), pred_step) + x = np.hstack(x_cols) + y = y[seq_len + pred_step - 1:] # make it fit with x sequences + + self.x = torch.from_numpy(x) + self.y = torch.from_numpy(y) self.seq_len = seq_len - self.pred_step = pred_step - self.size = self.x.shape[0] - seq_len - pred_step + 1 + self.size = self.y.shape[0] + self.transforms = transforms if self.size <= 0: raise ValueError(f"Error with given seq_len: {seq_len} and pred_step: {pred_step}" "dataset can't return any data (they are probably too big)") - if seq_len <= 0: - raise ValueError(f"seq_len must") + def __push_cols(self, cols: np.array, push_idx: List[int], d: int) -> np.array: + """ + pushes cols of indexes in push_idx by value d, + other cols get cutted by value d + """ + + if d == 0: + return cols - if pred_step < 0: - raise ValueError(f"pred_step can't be negative") + new_cols = [None] * len(cols) + + for i, _ in enumerate(cols): + if i in push_idx: + new_cols[i] = cols[i][d:] + else: + new_cols[i] = cols[i][:-d] + + return new_cols + + def get_weighted_rnd_sampler(self) -> WeightedRandomSampler: + """ + returns instance of torch.utils.data.WeightedRandomSampler + with weights so that weight[i] is set to probability density + function value for element returned by self.__getitem__[i][1] + """ + + unique, indexes, occurs = np.unique(self.y.numpy(), return_counts=True, return_inverse=True, axis=0) + weights = 1 / (occurs[indexes] * unique.shape[0]) + return WeightedRandomSampler(weights, self.size, True) - @override def __getitem__(self, idx) -> torch.tensor: """ returns tuple (seq, pred) where: @@ -76,10 +111,14 @@ class FCRdataSet(Dataset, ABC): pred is torch.tensor of shape (6,) """ - return (self.x[idx:idx + self.seq_len], - self.y[idx + self.seq_len + self.pred_step - 1]) + x = self.x[idx:idx + self.seq_len] + y = self.y[idx] + + if self.transforms is not None: + x = self.transforms(x) + + return (x, y) - @override def __len__(self): return self.size @@ -89,7 +128,6 @@ class FCRtrainDataSet(FCRdataSet): represents FCR dataset for training """ - @override def load_series(self, file_path: Path): series = np.genfromtxt( file_path, @@ -99,7 +137,8 @@ class FCRtrainDataSet(FCRdataSet): dtype=np.float32 ) test_s = test_size([series.shape[0]]) - return series[:-test_s] + rez = np.hsplit(series[:-test_s], [3]) + return rez class FCRtestDataSet(FCRdataSet): @@ -107,7 +146,6 @@ class FCRtestDataSet(FCRdataSet): represents FCR dataset for testing """ - @override def load_series(self, file_path: Path): series = np.genfromtxt( file_path, @@ -117,4 +155,4 @@ class FCRtestDataSet(FCRdataSet): dtype=np.float32 ) test_s = test_size([series.shape[0]]) - return series[-test_s:] + return np.hsplit(series[-test_s:], [3]) -- GitLab From 3b58c7e8cc1fcb954403ed0ac02cb2921c2a6dc8 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Thu, 14 Jan 2021 21:16:11 +0100 Subject: [PATCH 12/26] created uniform setup for developing FCR neural network. Includes testing, validation, and metric logging --- FCRtraining/README.md | 109 +++++++++++++++++++++++++ FCRtraining/metrics/RowAccuracy.py | 26 ++++++ FCRtraining/networks/LitFCRtestBase.py | 92 +++++++++++++++++++++ FCRtraining/networks/lstm_network1.py | 84 +++++++++++++++++++ FCRtraining/pathloader.py | 3 + FCRtraining/train.py | 9 ++ 6 files changed, 323 insertions(+) create mode 100644 FCRtraining/README.md create mode 100644 FCRtraining/metrics/RowAccuracy.py create mode 100644 FCRtraining/networks/LitFCRtestBase.py create mode 100644 FCRtraining/networks/lstm_network1.py create mode 100644 FCRtraining/pathloader.py create mode 100644 FCRtraining/train.py diff --git a/FCRtraining/README.md b/FCRtraining/README.md new file mode 100644 index 0000000..f687041 --- /dev/null +++ b/FCRtraining/README.md @@ -0,0 +1,109 @@ +# pytorch lightning training setup for FCR app +# How to install dependencies + - make sure you have latest version of pytorch and pytorch-lightning installed + +

+ +# Contents +## FCRtraining.networks.LitFCRtestBase.BaseTestEncoder + - subclass of pytorch_lightning.LightningModule + - implements testing and training metrics logging + - use it as base of your LightningModule class + - details about LightningModule: https://pytorch-lightning.readthedocs.io/en/stable/lightning_module.html + +

+ +## Training + - to start training or testing run python -m FCRtraining.train + - you can modify train.py freely for your flexibility + +

+ +# Example of BaseTestEncoder usage +```Python +from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet, FCRtestDataSet +from torch.utils.data import DataLoader +from torch.optim.lr_scheduler import ReduceLROnPlateau +from .LitFCRtestBase import BaseTestEncoder +import torch.nn as nn +import torch + + +HIDDEN_SIZE = 40 +BATCH_SIZE = 32 +SEQ_LEN = 10 +HORIZON = 0 +LR = 0.01 + +FEATURES = 3 +OUTPUT = 6 + + +class Encoder(BaseTestEncoder): + def __init__( + self, + features=FEATURES, + output=OUTPUT, + learning_rate=LR, + batch_size=BATCH_SIZE, + seq_len=SEQ_LEN, + horizon=HORIZON, + hidden_size=HIDDEN_SIZE, + + ): + super(Encoder, self).__init__() + + self.seq_len = seq_len + self.horizon = horizon + self.batch_size = batch_size + + self.criterion = nn.MSELoss() + self.lr = learning_rate + + self.lstm = nn.LSTM(features, hidden_size, num_layers=2, + bidirectional=True, batch_first=True) + self.fc = nn.Linear(hidden_size * 2, output) + + def forward(self, x): + out, _ = self.lstm(x) + # out: (batch, features, hidden_size * directions) + out = out[:, -1, :] + # out: (batch, hidden_size * directions) + out = self.fc(out) + return out + + def training_step(self, batch, batch_idx): + x, y = batch + prediction = self(x) + loss = self.criterion(prediction, y) + + self.log('train_loss', loss, on_step=False, on_epoch=True) + + return loss + + def val_dataloader(self): + return self.test_dataloader() + + def train_dataloader(self): + train_data = FCRtrainDataSet(self.seq_len, self.horizon) + loader = DataLoader(train_data, batch_size=self.batch_size, + num_workers=4)#, sampler=train_data.get_weighted_rnd_sampler()) + return loader + + def test_dataloader(self): + test_data = FCRtestDataSet(self.seq_len, self.horizon) + loader = DataLoader(test_data, batch_size=self.batch_size, + num_workers=4)#, sampler=test_data.get_weighted_rnd_sampler()) + return loader + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.lr) + scheduler = ReduceLROnPlateau( + optimizer, 'min', patience=10, verbose=True) + return { + 'optimizer': optimizer, + 'lr_scheduler': scheduler, + 'monitor': 'train_loss' + } + +``` \ No newline at end of file diff --git a/FCRtraining/metrics/RowAccuracy.py b/FCRtraining/metrics/RowAccuracy.py new file mode 100644 index 0000000..e2bb468 --- /dev/null +++ b/FCRtraining/metrics/RowAccuracy.py @@ -0,0 +1,26 @@ +from pytorch_lightning.metrics import Metric +import torch + + +class RowAccuracy(Metric): + """ + Represents Accuracy of matches in dim=1 in givens tensors + + implemented as in: + https://pytorch-lightning.readthedocs.io/en/latest/metrics.html + """ + + def __init__(self, dist_sync_on_step=False): + super().__init__(dist_sync_on_step=dist_sync_on_step) + + self.add_state("correct", default=torch.tensor(0), dist_reduce_fx="sum") + self.add_state("total", default=torch.tensor(0), dist_reduce_fx="sum") + + def update(self, preds: torch.Tensor, target: torch.Tensor): + preds, target = preds, target + assert preds.shape == target.shape + self.correct += (preds == target).all(dim=1).sum() # count all row matches + self.total += target.shape[0] # add batch size + + def compute(self): + return self.correct.float() / self.total diff --git a/FCRtraining/networks/LitFCRtestBase.py b/FCRtraining/networks/LitFCRtestBase.py new file mode 100644 index 0000000..64100ee --- /dev/null +++ b/FCRtraining/networks/LitFCRtestBase.py @@ -0,0 +1,92 @@ +from ..metrics.RowAccuracy import RowAccuracy +import pytorch_lightning as pl +import torch.nn as nn +import torch + + +class BaseTestEncoder(pl.LightningModule): + """ + abstract base class for LightningModule, + implements validation and test loops including logging + subclass must implement criterion as loss function + """ + + def __init__(self): + ''' + creates train, and test metrics + TODO: add metric from smoteR paper + ''' + + super(BaseTestEncoder, self).__init__() + + # train metrics + self.train_rounded_accuracy = RowAccuracy() + self.train_rounded_mse = pl.metrics.MeanSquaredError() + self.train_rounded_mae = pl.metrics.MeanAbsoluteError() + self.train_mse = pl.metrics.MeanSquaredError() + self.train_mae = pl.metrics.MeanAbsoluteError() + + # test metrics + self.test_rounded_accuracy = RowAccuracy() + self.test_rounded_mse = pl.metrics.MeanSquaredError() + self.test_rounded_mae = pl.metrics.MeanAbsoluteError() + self.test_mse = pl.metrics.MeanSquaredError() + self.test_mae = pl.metrics.MeanAbsoluteError() + + def validation_step(self, batch, batch_nb): + """ + predicts y, and calculates loss in training + """ + + x, y = batch + preds = self(x) + loss = self.criterion(preds, y) # might not be necessary + + return {'loss': loss, 'preds': preds, 'target': y} + + def validation_step_end(self, outputs): + ''' + update and log validation metrics + ''' + + rounded_preds = torch.round(outputs['preds']) + self.train_rounded_accuracy(rounded_preds, outputs['target']) + self.train_rounded_mse(rounded_preds, outputs['target']) + self.train_rounded_mae(rounded_preds, outputs['target']) + self.train_mse(outputs['preds'], outputs['target']) + self.train_mae(outputs['preds'], outputs['target']) + + self.log('train_rounded_accuracy', self.train_rounded_accuracy, on_step=False, on_epoch=True) + self.log('train_rounded_mse', self.train_rounded_mse, on_step=False, on_epoch=True) + self.log('train_rounded_mae', self.train_rounded_mae, on_step=False, on_epoch=True) + self.log('train_mse', self.train_mse, on_step=False, on_epoch=True) + self.log('train_mae', self.train_mae, on_step=False, on_epoch=True) + + def test_step(self, batch, batch_idx): + """ + predicts y, and calculates loss in testing + """ + + x, y = batch + preds = self(x) + loss = self.criterion(preds, y) # might not be necessary + + return {'loss': loss, 'preds': preds, 'target': y} + + def test_step_end(self, outputs): + ''' + update and log test metrics + ''' + + rounded_preds = torch.round(outputs['preds']) + self.test_rounded_accuracy(rounded_preds, outputs['target']) + self.test_rounded_mse(rounded_preds, outputs['target']) + self.test_rounded_mae(rounded_preds, outputs['target']) + self.test_mse(outputs['preds'], outputs['target']) + self.test_mae(outputs['preds'], outputs['target']) + + self.log('test_rounded_accuracy', self.test_rounded_accuracy, on_step=False, on_epoch=True) + self.log('test_rounded_mse', self.test_rounded_mse, on_step=False, on_epoch=True) + self.log('test_rounded_mae', self.test_rounded_mae, on_step=False, on_epoch=True) + self.log('test_mse', self.test_mse, on_step=False, on_epoch=True) + self.log('test_mae', self.test_mae, on_step=False, on_epoch=True) diff --git a/FCRtraining/networks/lstm_network1.py b/FCRtraining/networks/lstm_network1.py new file mode 100644 index 0000000..943db17 --- /dev/null +++ b/FCRtraining/networks/lstm_network1.py @@ -0,0 +1,84 @@ +from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet, FCRtestDataSet +from torch.utils.data import DataLoader +from torch.optim.lr_scheduler import ReduceLROnPlateau +from .LitFCRtestBase import BaseTestEncoder +import torch.nn as nn +import torch + + +HIDDEN_SIZE = 40 +BATCH_SIZE = 32 +SEQ_LEN = 10 +HORIZON = 0 +LR = 0.01 + +FEATURES = 3 +OUTPUT = 6 + + +class Encoder(BaseTestEncoder): + def __init__( + self, + features=FEATURES, + output=OUTPUT, + learning_rate=LR, + batch_size=BATCH_SIZE, + seq_len=SEQ_LEN, + horizon=HORIZON, + hidden_size=HIDDEN_SIZE, + + ): + super(Encoder, self).__init__() + + self.seq_len = seq_len + self.horizon = horizon + self.batch_size = batch_size + + self.criterion = nn.MSELoss() + self.lr = learning_rate + + self.lstm = nn.LSTM(features, hidden_size, num_layers=2, + bidirectional=True, batch_first=True) + self.fc = nn.Linear(hidden_size * 2, output) + + def forward(self, x): + out, _ = self.lstm(x) + # out: (batch, features, hidden_size * directions) + out = out[:, -1, :] + # out: (batch, hidden_size * directions) + out = self.fc(out) + return out + + def training_step(self, batch, batch_idx): + x, y = batch + prediction = self(x) + loss = self.criterion(prediction, y) + + self.log('train_loss', loss, on_step=False, on_epoch=True) + + return loss + + def val_dataloader(self): + return self.test_dataloader() + + def train_dataloader(self): + train_data = FCRtrainDataSet(self.seq_len, self.horizon) + loader = DataLoader(train_data, batch_size=self.batch_size, + num_workers=4) + return loader + + def test_dataloader(self): + test_data = FCRtestDataSet(self.seq_len, self.horizon) + loader = DataLoader(test_data, batch_size=self.batch_size, + num_workers=4) + return loader + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.lr) + scheduler = ReduceLROnPlateau( + optimizer, 'min', patience=10, verbose=True) + return { + 'optimizer': optimizer, + 'lr_scheduler': scheduler, + 'monitor': 'train_loss' + } diff --git a/FCRtraining/pathloader.py b/FCRtraining/pathloader.py new file mode 100644 index 0000000..ce2bfec --- /dev/null +++ b/FCRtraining/pathloader.py @@ -0,0 +1,3 @@ +import sys +import os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) \ No newline at end of file diff --git a/FCRtraining/train.py b/FCRtraining/train.py new file mode 100644 index 0000000..da7a3f4 --- /dev/null +++ b/FCRtraining/train.py @@ -0,0 +1,9 @@ +from . import pathloader +from .networks.lstm_network1 import Encoder +import pytorch_lightning as pl + + +model = Encoder(batch_size=64) +trainer = pl.Trainer(min_epochs=1, max_epochs=100) +trainer.fit(model) +trainer.test(model) -- GitLab From 7d14b668c8b4242fb28cc7c2ce60a0217583efbf Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Thu, 14 Jan 2021 21:20:31 +0100 Subject: [PATCH 13/26] readmue update --- FCRtraining/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/FCRtraining/README.md b/FCRtraining/README.md index f687041..9d83e01 100644 --- a/FCRtraining/README.md +++ b/FCRtraining/README.md @@ -19,6 +19,15 @@

+## Metrics logging + - after running experiment directory `lightning_logs` should appear + - you can inspect logs with `tensorboard` by running: +```sh +$ tensorboard --logdir= +``` + +

+ # Example of BaseTestEncoder usage ```Python from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet, FCRtestDataSet -- GitLab From 1305f878f4369e2455729c2f7ff077fe5c4b6cfd Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Thu, 14 Jan 2021 21:26:51 +0100 Subject: [PATCH 14/26] removed local file, updated README --- FCRtraining/README.md | 8 -------- FCRtraining/train.py | 9 --------- 2 files changed, 17 deletions(-) delete mode 100644 FCRtraining/train.py diff --git a/FCRtraining/README.md b/FCRtraining/README.md index 9d83e01..1abe27b 100644 --- a/FCRtraining/README.md +++ b/FCRtraining/README.md @@ -13,12 +13,6 @@

-## Training - - to start training or testing run python -m FCRtraining.train - - you can modify train.py freely for your flexibility - -

- ## Metrics logging - after running experiment directory `lightning_logs` should appear - you can inspect logs with `tensorboard` by running: @@ -26,8 +20,6 @@ $ tensorboard --logdir= ``` -

- # Example of BaseTestEncoder usage ```Python from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet, FCRtestDataSet diff --git a/FCRtraining/train.py b/FCRtraining/train.py deleted file mode 100644 index da7a3f4..0000000 --- a/FCRtraining/train.py +++ /dev/null @@ -1,9 +0,0 @@ -from . import pathloader -from .networks.lstm_network1 import Encoder -import pytorch_lightning as pl - - -model = Encoder(batch_size=64) -trainer = pl.Trainer(min_epochs=1, max_epochs=100) -trainer.fit(model) -trainer.test(model) -- GitLab From 339eb93cc52e78ff00b9f54d8d414c33b63a7c3e Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Mon, 18 Jan 2021 01:20:22 +0100 Subject: [PATCH 15/26] big modifications to dataset, created SequenceForecastMultiDistributionDatasetFactory, that creates pytorch datasets based on set of csv files --- FCRdataLoader/fcrdataloader/dataset.py | 247 ++++++++++++++++++------- 1 file changed, 176 insertions(+), 71 deletions(-) diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py index 9e8f528..9576911 100644 --- a/FCRdataLoader/fcrdataloader/dataset.py +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -1,54 +1,98 @@ -from torch.utils.data import Dataset, WeightedRandomSampler +from torch.utils.data import Dataset, WeightedRandomSampler, DataLoader from pathlib import Path -from .data import DATA_FILES -from typing import List +from typing import List, Callable from abc import ABC, abstractmethod import numpy as np import torch MAX_TEST_S = 1000 -TEST_TRAIN_RATIO = 0.4 +TEST_TRAIN_RATIO = 0.3 -def test_size(file_sizes: List[int]) -> int: +class SequenceForecastMultiDistributionDataset(Dataset): """ - given list of sizes of datasets in datafiles returns test size + Pytorch subclass of dataset, represents dataset of + mapping sequence of inputs to single output with + some horizon. Also dataset is structured as set of + smaller datasets. It makes difference, with sequence + mapping becouse "boundaries" of datasets can not be + considered as valid sequence. """ - total = sum(file_sizes) - test_s = min(MAX_TEST_S, int(TEST_TRAIN_RATIO * total)) - return test_s + def __init__( + self, + start: int, + size: int, + getitem: Callable[[int], torch.Tensor], + ): + self.start = start + self.size = size + self.getitem = getitem + def __len__(self): + return self.size + + def __getitem__(self, idx): + return self.getitem(self.start + idx) -class FCRdataSet(Dataset, ABC): + +class SequenceForecastMultiDistributionDatasetFactory(ABC): """ + TODO update docs + TODO add input validation + TODO add tests??? abstract class representing FCR dataset Currently only uses data from one file + + must return tuple (x, y) where: + LEGEND: + t - timestamp + v(t) - avg rsp time at time t + p(t) - avg rsp time prediction at time t + split(t) - split value corresponding to prediction at time t + solv(t) - solution to cp-problem at time t (obtained from v(t)) + TUPLE: + x is np.array od dtype: float32 with values (v(t), p(t), split(t)), shape (n, 3) + y is np.array od dtype: float32 with values (solv(t)), shape (n, 6) """ - @abstractmethod - def load_series(self, file_path: Path) -> (np.array, np.array): + def __test_size(self, rows: int) -> int: """ - must return tuple (x, y) where: - LEGEND: - t - timestamp - v(t) - avg rsp time at time t - p(t) - avg rsp time prediction at time t - split(t) - split value corresponding to prediction at time t - solv(t) - solution to cp-problem at time t (obtained from v(t)) - TUPLE: - x is np.array od dtype: float32 with values (v(t), p(t), split(t)), shape (n, 3) - y is np.array od dtype: float32 with values (solv(t)), shape (n, 6) + given total rows of data returns test size """ - pass - def __init__(self, seq_len: int, pred_step: int, transforms=None): + return min(MAX_TEST_S, int(TEST_TRAIN_RATIO * rows)) + + def __train_size(self, rows: int) -> int: + """ + given total rows of data returns train size + """ + return rows - self.__test_size(rows) + + def __init__( + self, + seq_len: int, + pred_step: int, + files: List[str], + x_y_split: int, + usecols: List[int], + x_predictions_cols: List[int], + transforms=None, + delimiter=',', + skip_header=1 + ): """ seq_len: is number of following records retuned as a sequence pred_step: is distance between last record in sequence and correctly predicted row transforms: transformation applited input data. if returned data is (x, y) then with transormation its (transorms(x), y) + usecols should contain ids of colums to be read from csv + in cols [0, x_y_split-1] we have intput cols mappings + in cols [x_y_split, end-1] we have output cols mapping + x_predictions_cols is list of cols from input that contain some prediction value + and they will be pushed in horizon creating preprocessing + """ if seq_len < 0: @@ -57,22 +101,60 @@ class FCRdataSet(Dataset, ABC): if pred_step < 0: raise ValueError(f"pred_step can't be negative") - x, y = self.load_series(DATA_FILES[1]) # for now only one file - x_cols = np.hsplit(x, x.shape[1]) - - x_cols = self.__push_cols(x_cols, range(1, 3), pred_step) - x = np.hstack(x_cols) - y = y[seq_len + pred_step - 1:] # make it fit with x sequences - - self.x = torch.from_numpy(x) - self.y = torch.from_numpy(y) - self.seq_len = seq_len - self.size = self.y.shape[0] + self.x = [None] * len(files) + self.y = [None] * len(files) + self.files = files self.transforms = transforms + self.seq_len = seq_len + self.x_y_split = x_y_split + self.usecols = usecols + self.x_predictions_cols = x_predictions_cols + self.delimiter = delimiter + self.skip_header = skip_header + self.size = None - if self.size <= 0: - raise ValueError(f"Error with given seq_len: {seq_len} and pred_step: {pred_step}" - "dataset can't return any data (they are probably too big)") + self.__preproces_data(seq_len, pred_step) + + def load_series(self, file: str) -> (np.array, np.array): + series = np.genfromtxt( + file, + delimiter=self.delimiter, + skip_header=self.skip_header, + usecols=self.usecols, + dtype=np.float32 + ) + return np.hsplit(series, [self.x_y_split]) + + def __preproces_data(self, seq_len, pred_step): + sizes = np.zeros(len(self.files), dtype=np.int) + for i, f_path in enumerate(self.files): + x, y = self.load_series(f_path) # for now only one file + x_cols = np.hsplit(x, x.shape[1]) + + x_cols = self.__push_cols(x_cols, self.x_predictions_cols, pred_step) + x = np.hstack(x_cols) + y = y[seq_len + pred_step - 1:] # make it fit with x sequences + + self.x[i] = torch.from_numpy(x) + self.y[i] = torch.from_numpy(y) + sizes[i] = y.shape[0] + + if sizes[i] <= 0: + raise ValueError(f"Error with given seq_len: {seq_len} and pred_step: {pred_step} " + f"dataset can't return any data from file {f_path} (they are probably too big)") + + cumsizes = np.cumsum(sizes, dtype=np.int) + self.size = cumsizes[-1].item() + self.idx_file_map = np.searchsorted(cumsizes, np.arange(self.size) + 1).astype(np.int) + self.idx_file_idx_map = np.zeros(self.size).astype(np.int) + + self.idx_file_idx_map[0] = 0 + cnt = 1 + for i in range(1, len(self.idx_file_map)): + if self.idx_file_map[i] != self.idx_file_map[i-1]: + cnt = 0 + self.idx_file_idx_map[i] = cnt + cnt += 1 def __push_cols(self, cols: np.array, push_idx: List[int], d: int) -> np.array: """ @@ -96,63 +178,86 @@ class FCRdataSet(Dataset, ABC): def get_weighted_rnd_sampler(self) -> WeightedRandomSampler: """ returns instance of torch.utils.data.WeightedRandomSampler - with weights so that weight[i] is set to probability density - function value for element returned by self.__getitem__[i][1] + with weights so that after applying it to as sampler in + pytorch dataloader, with train_dataset output mapping + will have almost uniform distribution """ - unique, indexes, occurs = np.unique(self.y.numpy(), return_counts=True, return_inverse=True, axis=0) + train_ds = self.get_train_dataset() + loader = DataLoader(train_set, batch_size=len(train_ds)) + unique, indexes, occurs = np.unique( + next(iter(loader))[1], # get all "y" from train_dataset (outputs) + return_counts=True, + return_inverse=True, + axis=0 + ) weights = 1 / (occurs[indexes] * unique.shape[0]) return WeightedRandomSampler(weights, self.size, True) - def __getitem__(self, idx) -> torch.tensor: + def __getitem(self, idx) -> torch.tensor: """ + TODO update this docstr returns tuple (seq, pred) where: seq is torch.tensor of shape (seq_len, 3) pred is torch.tensor of shape (6,) """ - x = self.x[idx:idx + self.seq_len] - y = self.y[idx] + f = self.idx_file_map[idx] + f_idx = self.idx_file_idx_map[idx] + x = self.x[f][f_idx:f_idx + self.seq_len] + y = self.y[f][f_idx] if self.transforms is not None: x = self.transforms(x) return (x, y) - def __len__(self): - return self.size + def __get_dataset(self, start, size): + return SequenceForecastMultiDistributionDataset( + start, + size, + self.__getitem + ) + + def get_train_dataset(self) -> SequenceForecastMultiDistributionDataset: + return self.__get_dataset(0, self.__train_size(self.size)) + def get_test_set(self) -> SequenceForecastMultiDistributionDataset: + return self.__get_dataset(self.__test_size(self.size), self.size) -class FCRtrainDataSet(FCRdataSet): + +class FCRdatasetFactory(SequenceForecastMultiDistributionDatasetFactory): """ - represents FCR dataset for training + FCR dataset factory """ - def load_series(self, file_path: Path): - series = np.genfromtxt( - file_path, - delimiter=',', - skip_header=1, + def __init__( + self, + seq_len: int, + pred_step: int, + transforms=None, + ): + from .data import DATA_FILES + + super().__init__( + seq_len=seq_len, + pred_step=pred_step, + transforms=transforms, + files=DATA_FILES, + x_y_split=3, usecols=range(1, 10), - dtype=np.float32 + x_predictions_cols=range(1, 3) ) - test_s = test_size([series.shape[0]]) - rez = np.hsplit(series[:-test_s], [3]) - return rez -class FCRtestDataSet(FCRdataSet): - """ - represents FCR dataset for testing - """ - def load_series(self, file_path: Path): - series = np.genfromtxt( - file_path, - delimiter=',', - skip_header=1, - usecols=range(1, 10), - dtype=np.float32 - ) - test_s = test_size([series.shape[0]]) - return np.hsplit(series[-test_s:], [3]) +from torch.utils.data import DataLoader +from collections import Counter + + +FCRdsFactory = FCRdatasetFactory(115, 0) +train_set = FCRdsFactory.get_train_dataset() +loader = DataLoader(train_set, batch_size=len(train_set), sampler=FCRdsFactory.get_weighted_rnd_sampler()) + +cnt = Counter(map(lambda x: tuple(x.numpy()), next(iter(loader))[1])) +print(cnt.values()) -- GitLab From 5664c353ad0040dd9d9d64b54231ee62eb2906c9 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Mon, 18 Jan 2021 11:49:17 +0100 Subject: [PATCH 16/26] added simple unit tests, docs still need to be updated --- FCRdataLoader/data/data1.csv | 998 -------------------- FCRdataLoader/fcrdataloader/dataset.py | 28 +- FCRdataLoader/setup.py | 1 + FCRdataLoader/tests/__init__.py | 0 FCRdataLoader/tests/seq1pred0_test.py | 35 + FCRdataLoader/tests/test_data/fcrdata10.csv | 12 + FCRdataLoader/tests/test_data/fcrdata2.csv | 11 + 7 files changed, 70 insertions(+), 1015 deletions(-) delete mode 100644 FCRdataLoader/data/data1.csv create mode 100644 FCRdataLoader/tests/__init__.py create mode 100644 FCRdataLoader/tests/seq1pred0_test.py create mode 100644 FCRdataLoader/tests/test_data/fcrdata10.csv create mode 100644 FCRdataLoader/tests/test_data/fcrdata2.csv diff --git a/FCRdataLoader/data/data1.csv b/FCRdataLoader/data/data1.csv deleted file mode 100644 index 8e076fc..0000000 --- a/FCRdataLoader/data/data1.csv +++ /dev/null @@ -1,998 +0,0 @@ -timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB -2020-10-29T14:06:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:07:10.029,0.0,8.092933654785156,0,1,0,1,0,1,0 -2020-10-29T14:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:09:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:10:10.029,0.0,9.235248565673828,0,1,0,1,0,1,0 -2020-10-29T14:11:10.029,10.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:12:10.029,0.0,12.410219192504883,0,1,0,1,0,1,0 -2020-10-29T14:13:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:14:10.029,0.0,8.37138557434082,0,1,0,1,0,1,0 -2020-10-29T14:15:10.029,20.0,0.07390737533569336,0,1,0,1,0,1,0 -2020-10-29T14:16:10.029,6.0,20.59613037109375,0,1,0,1,0,1,0 -2020-10-29T14:17:10.029,0.0,1.4238762855529785,0,1,0,1,0,1,0 -2020-10-29T14:18:10.029,14.0,0.4126596450805664,0,1,0,1,0,1,0 -2020-10-29T14:19:10.029,3.0,20.156076431274414,0,1,0,1,0,1,0 -2020-10-29T14:20:10.029,0.0,3.812490463256836,0,1,0,1,0,1,0 -2020-10-29T14:21:10.029,0.0,1.0089221000671387,0,1,0,1,0,1,0 -2020-10-29T14:22:10.029,0.0,0.984745979309082,0,1,0,1,0,1,0 -2020-10-29T14:23:10.029,0.0,0.3623847961425781,0,1,0,1,0,1,0 -2020-10-29T14:24:10.029,0.0,0.0015397071838378906,0,1,0,1,0,1,0 -2020-10-29T14:25:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:26:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:27:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:28:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:29:10.029,8.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:30:10.029,0.0,8.296137809753418,0,1,0,1,0,1,0 -2020-10-29T14:31:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:32:10.029,0.0,5.666323661804199,0,1,0,1,0,1,0 -2020-10-29T14:33:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:34:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:35:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:36:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:37:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:38:10.029,26.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:39:10.029,0.0,10.709664344787598,0,1,0,1,0,1,0 -2020-10-29T14:40:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:41:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:42:10.029,0.0,5.936244010925293,0,1,0,1,0,1,0 -2020-10-29T14:43:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:44:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:45:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:46:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:47:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:48:10.029,9.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:49:10.029,0.0,7.949969291687012,0,1,0,1,0,1,0 -2020-10-29T14:50:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:51:10.029,342.0,0.0,0,7,0,1,0,1,0 -2020-10-29T14:52:10.029,0.0,21.785860061645508,0,1,0,1,0,1,0 -2020-10-29T14:53:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:54:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:55:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:56:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:57:10.029,9.0,0.0,0,1,0,1,0,1,0 -2020-10-29T14:58:10.029,0.0,6.099068641662598,0,1,0,1,0,1,0 -2020-10-29T14:59:10.029,11.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:00:10.029,5.0,8.87404727935791,0,1,0,1,0,1,0 -2020-10-29T15:01:10.029,7.0,0.14431333541870117,0,1,0,1,0,1,0 -2020-10-29T15:02:10.029,0.0,3.6838326454162598,0,1,0,1,0,1,0 -2020-10-29T15:03:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:04:10.029,5.0,6.429187774658203,0,1,0,1,0,1,0 -2020-10-29T15:05:10.029,0.0,2.1575698852539062,0,1,0,1,0,1,0 -2020-10-29T15:06:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:07:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:09:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:10:10.029,0.0,11.564858436584473,0,1,0,1,0,1,0 -2020-10-29T15:11:10.029,0.0,0.25743913650512695,0,1,0,1,0,1,0 -2020-10-29T15:12:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:13:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:14:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:15:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:16:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:17:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:18:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:19:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:20:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:21:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:22:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:23:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:24:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:25:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:26:10.029,0.0,15.251335144042969,0,1,0,1,0,1,0 -2020-10-29T15:27:10.029,356.0,0.0,0,7,0,1,0,1,0 -2020-10-29T15:28:10.029,0.0,46.44401550292969,0,1,0,1,0,1,0 -2020-10-29T15:29:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:30:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:31:10.029,6.0,1.699239730834961,0,1,0,1,0,1,0 -2020-10-29T15:32:10.029,0.0,1.8481130599975586,0,1,0,1,0,1,0 -2020-10-29T15:33:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:34:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:35:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:36:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:37:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:38:10.029,0.0,2.3151702880859375,0,1,0,1,0,1,0 -2020-10-29T15:39:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:40:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:41:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:42:10.029,15.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:43:10.029,23.0,11.84577751159668,0,1,0,1,0,1,0 -2020-10-29T15:44:10.029,0.0,13.07125473022461,0,1,0,1,0,1,0 -2020-10-29T15:45:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:46:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:47:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:48:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:49:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:50:10.029,0.0,5.202688217163086,0,1,0,1,0,1,0 -2020-10-29T15:51:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:52:10.029,50.0,6.235217094421387,0,1,0,1,0,1,0 -2020-10-29T15:53:10.029,5.0,25.572189331054688,0,1,0,1,0,1,0 -2020-10-29T15:54:10.029,28.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:55:10.029,0.0,19.234643936157227,0,1,0,1,0,1,0 -2020-10-29T15:56:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:57:10.029,0.0,6.527838706970215,0,1,0,1,0,1,0 -2020-10-29T15:58:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T15:59:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:00:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:01:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:02:10.029,13.0,11.12646770477295,0,1,0,1,0,1,0 -2020-10-29T16:03:10.029,8.0,12.76125717163086,0,1,0,1,0,1,0 -2020-10-29T16:04:10.029,0.0,4.127845764160156,0,1,0,1,0,1,0 -2020-10-29T16:05:10.029,0.0,0.1368122100830078,0,1,0,1,0,1,0 -2020-10-29T16:06:10.029,3.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:07:10.029,0.0,4.626254081726074,0,1,0,1,0,1,0 -2020-10-29T16:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:09:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:10:10.029,0.0,6.42384147644043,0,1,0,1,0,1,0 -2020-10-29T16:11:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:12:10.029,6.0,7.250375747680664,0,1,0,1,0,1,0 -2020-10-29T16:13:10.034,6.0,7.434720039367676,0,1,0,1,0,1,0 -2020-10-29T16:14:10.029,0.0,7.118711471557617,0,1,0,1,0,1,0 -2020-10-29T16:15:10.029,0.0,0.36592769622802734,0,1,0,1,0,1,0 -2020-10-29T16:16:10.029,4.5,0.0,0,1,0,1,0,1,0 -2020-10-29T16:17:10.029,5.0,7.875121116638184,0,1,0,1,0,1,0 -2020-10-29T16:18:10.029,6.5,6.107962608337402,0,1,0,1,0,1,0 -2020-10-29T16:19:10.029,0.0,8.611654281616211,0,1,0,1,0,1,0 -2020-10-29T16:20:10.029,0.0,0.5526518821716309,0,1,0,1,0,1,0 -2020-10-29T16:21:10.029,6.0,0.2611522674560547,0,1,0,1,0,1,0 -2020-10-29T16:22:10.029,4.0,11.57531452178955,0,1,0,1,0,1,0 -2020-10-29T16:23:10.029,5.0,3.7174105644226074,0,1,0,1,0,1,0 -2020-10-29T16:24:10.029,0.0,8.499622344970703,0,1,0,1,0,1,0 -2020-10-29T16:25:10.029,0.0,1.018892765045166,0,1,0,1,0,1,0 -2020-10-29T16:26:10.029,7.0,0.36528682708740234,0,1,0,1,0,1,0 -2020-10-29T16:27:10.029,5.0,11.12242317199707,0,1,0,1,0,1,0 -2020-10-29T16:28:10.029,0.0,4.008762359619141,0,1,0,1,0,1,0 -2020-10-29T16:29:10.029,0.0,0.9459342956542969,0,1,0,1,0,1,0 -2020-10-29T16:30:10.029,6.0,0.38293981552124023,0,1,0,1,0,1,0 -2020-10-29T16:31:10.029,0.0,10.566802978515625,0,1,0,1,0,1,0 -2020-10-29T16:32:10.029,0.0,0.2418060302734375,0,1,0,1,0,1,0 -2020-10-29T16:33:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:34:10.029,0.0,8.920586585998535,0,1,0,1,0,1,0 -2020-10-29T16:35:10.029,8.0,0.17075300216674805,0,1,0,1,0,1,0 -2020-10-29T16:36:10.029,0.0,11.595227241516113,0,1,0,1,0,1,0 -2020-10-29T16:37:10.029,0.0,0.06052255630493164,0,1,0,1,0,1,0 -2020-10-29T16:38:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:39:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:40:10.029,0.0,6.924949645996094,0,1,0,1,0,1,0 -2020-10-29T16:41:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:42:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:43:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:44:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:45:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:46:10.029,12.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:47:10.029,9.0,11.044121742248535,0,1,0,1,0,1,0 -2020-10-29T16:48:10.029,7.0,5.830473899841309,0,1,0,1,0,1,0 -2020-10-29T16:49:10.029,0.0,4.056408882141113,0,1,0,1,0,1,0 -2020-10-29T16:50:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:51:10.029,0.0,6.529085159301758,0,1,0,1,0,1,0 -2020-10-29T16:52:10.029,8.0,0.0,0,1,0,1,0,1,0 -2020-10-29T16:53:10.029,5.0,9.057021141052246,0,1,0,1,0,1,0 -2020-10-29T16:54:10.029,0.0,1.7574710845947266,0,1,0,1,0,1,0 -2020-10-29T16:55:10.029,202.0,0.0,0,4,0,1,0,1,0 -2020-10-29T16:56:10.029,52.0,33.17987060546875,0,1,0,1,0,1,0 -2020-10-29T16:57:10.029,0.0,4.32895565032959,0,1,0,1,0,1,0 -2020-10-29T16:58:10.029,0.0,0.5938620567321777,0,1,0,1,0,1,0 -2020-10-29T16:59:10.029,5.0,0.6138205528259277,0,1,0,1,0,1,0 -2020-10-29T17:00:10.029,0.0,5.230597496032715,0,1,0,1,0,1,0 -2020-10-29T17:01:10.029,0.0,0.027581214904785156,0,1,0,1,0,1,0 -2020-10-29T17:02:10.029,8.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:03:10.029,5.0,9.127798080444336,0,1,0,1,0,1,0 -2020-10-29T17:04:10.029,20.0,2.4755353927612305,0,1,0,1,0,1,0 -2020-10-29T17:05:10.029,7.0,20.369178771972656,0,1,0,1,0,1,0 -2020-10-29T17:06:10.029,6.0,3.315825939178467,0,1,0,1,0,1,0 -2020-10-29T17:07:10.029,0.0,4.419093132019043,0,1,0,1,0,1,0 -2020-10-29T17:08:10.029,5.0,0.1558208465576172,0,1,0,1,0,1,0 -2020-10-29T17:09:10.029,0.0,6.724039077758789,0,1,0,1,0,1,0 -2020-10-29T17:10:10.029,0.0,0.4746742248535156,0,1,0,1,0,1,0 -2020-10-29T17:11:10.029,0.0,0.15520572662353516,0,1,0,1,0,1,0 -2020-10-29T17:12:10.029,5.0,0.13993406295776367,0,1,0,1,0,1,0 -2020-10-29T17:13:10.029,5.0,10.159008026123047,0,1,0,1,0,1,0 -2020-10-29T17:14:10.029,0.0,5.231137275695801,0,1,0,1,0,1,0 -2020-10-29T17:15:10.029,0.0,0.714848518371582,0,1,0,1,0,1,0 -2020-10-29T17:16:10.029,4.0,0.1667160987854004,0,1,0,1,0,1,0 -2020-10-29T17:17:10.029,5.0,8.324742317199707,0,1,0,1,0,1,0 -2020-10-29T17:18:10.029,0.0,6.501380920410156,0,1,0,1,0,1,0 -2020-10-29T17:19:10.029,0.0,0.1724538803100586,0,1,0,1,0,1,0 -2020-10-29T17:20:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:21:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:22:10.029,13.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:23:10.029,0.0,13.301803588867188,0,1,0,1,0,1,0 -2020-10-29T17:24:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:25:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:26:10.029,0.0,7.142899513244629,0,1,0,1,0,1,0 -2020-10-29T17:27:10.029,251.0,0.0,0,5,0,1,0,1,0 -2020-10-29T17:28:10.029,0.0,23.88077163696289,0,1,0,1,0,1,0 -2020-10-29T17:29:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:30:10.029,8.5,0.0,0,1,0,1,0,1,0 -2020-10-29T17:31:10.029,4.0,7.784939765930176,0,1,0,1,0,1,0 -2020-10-29T17:32:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:33:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:34:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:35:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:36:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:37:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:38:10.029,6.0,5.221391677856445,0,1,0,1,0,1,0 -2020-10-29T17:39:10.029,0.0,1.9989275932312012,0,1,0,1,0,1,0 -2020-10-29T17:40:10.029,11.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:41:10.029,0.0,10.013413429260254,0,1,0,1,0,1,0 -2020-10-29T17:42:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:43:10.029,180.0,9.235176086425781,0,4,0,1,0,1,0 -2020-10-29T17:44:10.029,0.0,24.981861114501953,0,1,0,1,0,1,0 -2020-10-29T17:45:10.029,317.5,0.0,0,6,0,1,0,1,0 -2020-10-29T17:46:10.029,12.0,74.0953598022461,0,1,0,1,0,1,0 -2020-10-29T17:47:10.029,5.5,0.29892683029174805,0,1,0,1,0,1,0 -2020-10-29T17:48:10.029,5.5,3.1506519317626953,0,1,0,1,0,1,0 -2020-10-29T17:49:10.029,0.0,5.2662763595581055,0,1,0,1,0,1,0 -2020-10-29T17:50:10.029,0.0,1.3602228164672852,0,1,0,1,0,1,0 -2020-10-29T17:51:10.029,0.0,0.6114144325256348,0,1,0,1,0,1,0 -2020-10-29T17:52:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:53:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:54:10.029,0.0,1.7974481582641602,0,1,0,1,0,1,0 -2020-10-29T17:55:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:56:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:57:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:58:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T17:59:10.029,0.0,6.526606559753418,0,1,0,1,0,1,0 -2020-10-29T18:00:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:01:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:02:10.029,0.0,5.383490562438965,0,1,0,1,0,1,0 -2020-10-29T18:03:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:04:10.029,8.0,8.05236530303955,0,1,0,1,0,1,0 -2020-10-29T18:05:10.029,0.0,8.077051162719727,0,1,0,1,0,1,0 -2020-10-29T18:06:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:07:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:09:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:10:10.032,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:11:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:12:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:13:10.029,12.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:14:10.029,0.0,11.618339538574219,0,1,0,1,0,1,0 -2020-10-29T18:15:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:16:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:17:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:18:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:19:10.029,4.0,5.411664009094238,0,1,0,1,0,1,0 -2020-10-29T18:20:10.029,0.0,2.1512250900268555,0,1,0,1,0,1,0 -2020-10-29T18:21:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:22:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:23:10.029,162.0,0.0,0,3,0,1,0,1,0 -2020-10-29T18:24:10.029,4.0,17.396459579467773,0,1,0,1,0,1,0 -2020-10-29T18:25:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:26:10.029,7.0,2.2944464683532715,0,1,0,1,0,1,0 -2020-10-29T18:27:10.029,0.0,4.356158256530762,0,1,0,1,0,1,0 -2020-10-29T18:28:10.029,16.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:29:10.029,0.0,14.886369705200195,0,1,0,1,0,1,0 -2020-10-29T18:30:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:31:10.029,0.0,5.8511962890625,0,1,0,1,0,1,0 -2020-10-29T18:32:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:33:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:34:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:35:10.029,0.0,3.7542366981506348,0,1,0,1,0,1,0 -2020-10-29T18:36:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:37:10.029,252.0,0.0,0,5,0,1,0,1,0 -2020-10-29T18:38:10.029,5.0,29.539676666259766,0,1,0,1,0,1,0 -2020-10-29T18:39:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:40:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:41:10.029,0.0,5.665234565734863,0,1,0,1,0,1,0 -2020-10-29T18:42:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:43:10.029,5.5,0.0,0,1,0,1,0,1,0 -2020-10-29T18:44:10.029,5.0,4.7825517654418945,0,1,0,1,0,1,0 -2020-10-29T18:45:10.029,0.0,1.490556240081787,0,1,0,1,0,1,0 -2020-10-29T18:46:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:47:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:48:10.029,6.0,4.517246246337891,0,1,0,1,0,1,0 -2020-10-29T18:49:10.029,0.0,3.4358997344970703,0,1,0,1,0,1,0 -2020-10-29T18:50:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:51:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:52:10.029,3.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:53:10.029,7.0,6.487791061401367,0,1,0,1,0,1,0 -2020-10-29T18:54:10.031,0.0,8.03091812133789,0,1,0,1,0,1,0 -2020-10-29T18:55:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T18:56:10.029,0.0,9.796905517578125,0,1,0,1,0,1,0 -2020-10-29T18:57:10.029,7.0,0.17009544372558594,0,1,0,1,0,1,0 -2020-10-29T18:58:10.029,4.0,10.307432174682617,0,1,0,1,0,1,0 -2020-10-29T18:59:10.029,0.0,2.21535062789917,0,1,0,1,0,1,0 -2020-10-29T19:00:10.029,0.0,0.5963382720947266,0,1,0,1,0,1,0 -2020-10-29T19:01:10.029,8.0,0.15051031112670898,0,1,0,1,0,1,0 -2020-10-29T19:02:10.029,9.0,11.291885375976562,0,1,0,1,0,1,0 -2020-10-29T19:03:10.029,5.0,9.199846267700195,0,1,0,1,0,1,0 -2020-10-29T19:04:10.029,0.0,3.387078285217285,0,1,0,1,0,1,0 -2020-10-29T19:05:10.029,6.0,1.3799810409545898,0,1,0,1,0,1,0 -2020-10-29T19:06:10.029,0.0,14.252178192138672,0,1,0,1,0,1,0 -2020-10-29T19:07:10.029,0.0,1.2471351623535156,0,1,0,1,0,1,0 -2020-10-29T19:08:10.029,6.0,0.6201796531677246,0,1,0,1,0,1,0 -2020-10-29T19:09:10.029,38.0,11.787551879882812,0,1,0,1,0,1,0 -2020-10-29T19:10:10.029,0.0,29.54815673828125,0,1,0,1,0,1,0 -2020-10-29T19:11:10.029,5.0,0.4530959129333496,0,1,0,1,0,1,0 -2020-10-29T19:12:10.029,0.0,9.565673828125,0,1,0,1,0,1,0 -2020-10-29T19:13:10.029,0.0,0.3695106506347656,0,1,0,1,0,1,0 -2020-10-29T19:14:10.029,0.0,0.36685609817504883,0,1,0,1,0,1,0 -2020-10-29T19:15:10.029,239.0,0.0,0,5,0,1,0,1,0 -2020-10-29T19:16:10.029,0.0,39.20317459106445,0,1,0,1,0,1,0 -2020-10-29T19:17:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:18:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:19:10.029,0.0,3.2760848999023438,0,1,0,1,0,1,0 -2020-10-29T19:20:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:21:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:22:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:23:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:24:10.029,8.0,3.594817638397217,0,1,0,1,0,1,0 -2020-10-29T19:25:10.029,0.0,3.721907615661621,0,1,0,1,0,1,0 -2020-10-29T19:26:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:27:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:28:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:29:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:30:10.029,5.0,4.781319618225098,0,1,0,1,0,1,0 -2020-10-29T19:31:10.029,0.0,4.42032527923584,0,1,0,1,0,1,0 -2020-10-29T19:32:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:33:10.029,0.0,6.181086540222168,0,1,0,1,0,1,0 -2020-10-29T19:34:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:35:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:36:10.029,0.0,8.427678108215332,0,1,0,1,0,1,0 -2020-10-29T19:37:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:38:10.029,8.0,9.44045352935791,0,1,0,1,0,1,0 -2020-10-29T19:39:10.029,0.0,7.80142879486084,0,1,0,1,0,1,0 -2020-10-29T19:40:10.029,0.0,0.18083858489990234,0,1,0,1,0,1,0 -2020-10-29T19:41:10.029,0.0,0.17366552352905273,0,1,0,1,0,1,0 -2020-10-29T19:42:10.029,0.0,0.16237449645996094,0,1,0,1,0,1,0 -2020-10-29T19:43:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:44:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:45:10.029,3.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:46:10.029,0.0,4.790645599365234,0,1,0,1,0,1,0 -2020-10-29T19:47:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:48:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:49:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:50:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:51:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:52:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:53:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:54:10.029,0.0,6.535590171813965,0,1,0,1,0,1,0 -2020-10-29T19:55:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:56:10.029,0.0,6.322650909423828,0,1,0,1,0,1,0 -2020-10-29T19:57:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:58:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T19:59:10.029,5.0,6.71759033203125,0,1,0,1,0,1,0 -2020-10-29T20:00:10.029,0.0,4.385467529296875,0,1,0,1,0,1,0 -2020-10-29T20:01:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:02:10.029,0.0,8.942854881286621,0,1,0,1,0,1,0 -2020-10-29T20:03:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:04:10.029,0.0,7.598843574523926,0,1,0,1,0,1,0 -2020-10-29T20:05:10.029,5.0,0.44580507278442383,0,1,0,1,0,1,0 -2020-10-29T20:06:10.029,7.0,10.363755226135254,0,1,0,1,0,1,0 -2020-10-29T20:07:10.029,0.0,9.921998023986816,0,1,0,1,0,1,0 -2020-10-29T20:08:10.029,6.0,1.1291399002075195,0,1,0,1,0,1,0 -2020-10-29T20:09:10.029,4.0,11.877375602722168,0,1,0,1,0,1,0 -2020-10-29T20:10:10.029,0.0,4.260271072387695,0,1,0,1,0,1,0 -2020-10-29T20:11:10.029,0.0,2.4076004028320312,0,1,0,1,0,1,0 -2020-10-29T20:12:10.029,0.0,1.1791324615478516,0,1,0,1,0,1,0 -2020-10-29T20:13:10.029,6.0,0.48076391220092773,0,1,0,1,0,1,0 -2020-10-29T20:14:10.029,6.0,10.632257461547852,0,1,0,1,0,1,0 -2020-10-29T20:15:10.029,0.0,7.27094841003418,0,1,0,1,0,1,0 -2020-10-29T20:16:10.029,0.0,0.6268486976623535,0,1,0,1,0,1,0 -2020-10-29T20:17:10.029,0.0,0.19138622283935547,0,1,0,1,0,1,0 -2020-10-29T20:18:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:19:10.029,0.0,7.540426254272461,0,1,0,1,0,1,0 -2020-10-29T20:20:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:21:10.029,4.5,0.0,0,1,0,1,0,1,0 -2020-10-29T20:22:10.029,0.0,6.261873245239258,0,1,0,1,0,1,0 -2020-10-29T20:23:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:24:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:25:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:26:10.029,0.0,7.58897590637207,0,1,0,1,0,1,0 -2020-10-29T20:27:10.029,40.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:28:10.029,0.0,14.05528450012207,0,1,0,1,0,1,0 -2020-10-29T20:29:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:30:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:31:10.029,15.0,8.196240425109863,0,1,0,1,0,1,0 -2020-10-29T20:32:10.029,0.0,11.093750953674316,0,1,0,1,0,1,0 -2020-10-29T20:33:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:34:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:35:10.029,0.0,7.550601959228516,0,1,0,1,0,1,0 -2020-10-29T20:36:10.029,8.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:37:10.029,0.0,12.223976135253906,0,1,0,1,0,1,0 -2020-10-29T20:38:10.029,0.0,0.39418792724609375,0,1,0,1,0,1,0 -2020-10-29T20:39:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:40:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:41:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:42:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:43:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:44:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:45:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:46:10.029,93.0,0.0,0,2,0,1,0,1,0 -2020-10-29T20:47:10.029,45.0,15.535181045532227,0,1,0,1,0,1,0 -2020-10-29T20:48:10.029,0.0,8.485981941223145,0,1,0,1,0,1,0 -2020-10-29T20:49:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:50:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:51:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:52:10.029,6.0,2.0895204544067383,0,1,0,1,0,1,0 -2020-10-29T20:53:10.029,0.0,1.9933748245239258,0,1,0,1,0,1,0 -2020-10-29T20:54:10.029,13.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:55:10.029,0.0,10.309151649475098,0,1,0,1,0,1,0 -2020-10-29T20:56:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:57:10.029,0.0,4.175436973571777,0,1,0,1,0,1,0 -2020-10-29T20:58:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T20:59:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:00:10.029,22.0,4.75355339050293,0,1,0,1,0,1,0 -2020-10-29T21:01:10.029,0.0,19.050439834594727,0,1,0,1,0,1,0 -2020-10-29T21:02:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:03:10.029,0.0,8.154370307922363,0,1,0,1,0,1,0 -2020-10-29T21:04:10.029,0.0,0.8469753265380859,0,1,0,1,0,1,0 -2020-10-29T21:05:10.029,0.0,0.34672975540161133,0,1,0,1,0,1,0 -2020-10-29T21:06:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:07:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:09:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:10:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:11:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:12:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:13:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:14:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:15:10.029,350.0,4.703037261962891,0,7,0,1,0,1,0 -2020-10-29T21:16:10.029,0.0,40.69681167602539,0,1,0,1,0,1,0 -2020-10-29T21:17:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:18:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:19:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:20:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:21:10.029,0.0,3.2666473388671875,0,1,0,1,0,1,0 -2020-10-29T21:22:10.030,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:23:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:24:10.029,0.0,3.8338260650634766,0,1,0,1,0,1,0 -2020-10-29T21:25:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:26:10.029,0.0,3.327702522277832,0,1,0,1,0,1,0 -2020-10-29T21:27:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:28:10.029,0.0,4.763402938842773,0,1,0,1,0,1,0 -2020-10-29T21:29:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:30:10.029,0.0,6.9983673095703125,0,1,0,1,0,1,0 -2020-10-29T21:31:10.029,0.0,0.13197612762451172,0,1,0,1,0,1,0 -2020-10-29T21:32:10.029,0.0,0.1350231170654297,0,1,0,1,0,1,0 -2020-10-29T21:33:10.029,9.0,0.11713886260986328,0,1,0,1,0,1,0 -2020-10-29T21:34:10.029,0.0,11.736916542053223,0,1,0,1,0,1,0 -2020-10-29T21:35:10.029,0.0,0.12194442749023438,0,1,0,1,0,1,0 -2020-10-29T21:36:10.029,0.0,0.11377096176147461,0,1,0,1,0,1,0 -2020-10-29T21:37:10.029,96.0,0.0,0,2,0,1,0,1,0 -2020-10-29T21:38:10.029,0.0,17.7043399810791,0,1,0,1,0,1,0 -2020-10-29T21:39:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:40:10.029,0.0,6.96132755279541,0,1,0,1,0,1,0 -2020-10-29T21:41:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:42:10.029,5.0,7.154741287231445,0,1,0,1,0,1,0 -2020-10-29T21:43:10.029,4.0,1.5402956008911133,0,1,0,1,0,1,0 -2020-10-29T21:44:10.029,0.0,1.4707465171813965,0,1,0,1,0,1,0 -2020-10-29T21:45:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:46:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:47:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:48:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:49:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:50:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:51:10.029,8.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:52:10.029,5.0,8.71021556854248,0,1,0,1,0,1,0 -2020-10-29T21:53:10.029,0.0,1.9077439308166504,0,1,0,1,0,1,0 -2020-10-29T21:54:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:55:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:56:10.029,22.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:57:10.029,0.0,9.41596508026123,0,1,0,1,0,1,0 -2020-10-29T21:58:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T21:59:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:00:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:01:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:02:10.029,0.0,5.084737777709961,0,1,0,1,0,1,0 -2020-10-29T22:03:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:04:10.029,0.0,7.03974723815918,0,1,0,1,0,1,0 -2020-10-29T22:05:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:06:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:07:10.029,0.0,6.234905242919922,0,1,0,1,0,1,0 -2020-10-29T22:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:09:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:10:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:11:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:12:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:13:10.029,15.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:14:10.029,0.0,10.901710510253906,0,1,0,1,0,1,0 -2020-10-29T22:15:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:16:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:17:10.029,0.0,5.1751298904418945,0,1,0,1,0,1,0 -2020-10-29T22:18:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:19:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:20:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:21:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:22:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:23:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:24:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:25:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:26:10.029,6.0,5.5879058837890625,0,1,0,1,0,1,0 -2020-10-29T22:27:10.029,0.0,4.2069902420043945,0,1,0,1,0,1,0 -2020-10-29T22:28:10.029,191.0,0.0,0,4,0,1,0,1,0 -2020-10-29T22:29:10.029,4.0,17.969911575317383,0,1,0,1,0,1,0 -2020-10-29T22:30:10.029,11.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:31:10.029,6.0,6.570542335510254,0,1,0,1,0,1,0 -2020-10-29T22:32:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:33:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:34:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:35:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:36:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:37:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:38:10.029,0.0,3.4944934844970703,0,1,0,1,0,1,0 -2020-10-29T22:39:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:40:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:41:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:42:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:43:10.029,0.0,4.713992118835449,0,1,0,1,0,1,0 -2020-10-29T22:44:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:45:10.029,0.0,6.209532737731934,0,1,0,1,0,1,0 -2020-10-29T22:46:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:47:10.029,3.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:48:10.029,0.0,4.264294624328613,0,1,0,1,0,1,0 -2020-10-29T22:49:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:50:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:51:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:52:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:53:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:54:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:55:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:56:10.030,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:57:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:58:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T22:59:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:00:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:01:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:02:10.029,287.0,16.195114135742188,0,6,0,1,0,1,0 -2020-10-29T23:03:10.029,0.0,38.41609191894531,0,1,0,1,0,1,0 -2020-10-29T23:04:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:05:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:06:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:07:10.029,0.0,2.750178813934326,0,1,0,1,0,1,0 -2020-10-29T23:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:09:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:10:10.029,0.0,2.201333522796631,0,1,0,1,0,1,0 -2020-10-29T23:11:10.029,12.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:12:10.029,0.0,8.798405647277832,0,1,0,1,0,1,0 -2020-10-29T23:13:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:14:10.029,59.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:15:10.029,0.0,22.150562286376953,0,1,0,1,0,1,0 -2020-10-29T23:16:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:17:10.029,0.0,6.523756980895996,0,1,0,1,0,1,0 -2020-10-29T23:18:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:19:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:20:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:21:10.029,0.0,6.732582092285156,0,1,0,1,0,1,0 -2020-10-29T23:22:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:23:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:24:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:25:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:26:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:27:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:28:10.029,6.0,4.772862434387207,0,1,0,1,0,1,0 -2020-10-29T23:29:10.029,0.0,5.109376907348633,0,1,0,1,0,1,0 -2020-10-29T23:30:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:31:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:32:10.029,0.0,4.709239959716797,0,1,0,1,0,1,0 -2020-10-29T23:33:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:34:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:35:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:36:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:37:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:38:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:39:10.029,0.0,4.721630096435547,0,1,0,1,0,1,0 -2020-10-29T23:40:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:41:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:42:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:43:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:44:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:45:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:46:10.029,0.0,4.526134490966797,0,1,0,1,0,1,0 -2020-10-29T23:47:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:48:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:49:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:50:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:51:10.029,0.0,6.65598201751709,0,1,0,1,0,1,0 -2020-10-29T23:52:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:53:10.029,10.5,4.829201698303223,0,1,0,1,0,1,0 -2020-10-29T23:54:10.029,0.0,8.031954765319824,0,1,0,1,0,1,0 -2020-10-29T23:55:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:56:10.029,0.0,6.480889320373535,0,1,0,1,0,1,0 -2020-10-29T23:57:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-29T23:58:10.029,0.0,8.18777084350586,0,1,0,1,0,1,0 -2020-10-29T23:59:10.029,5.0,0.05682659149169922,0,1,0,1,0,1,0 -2020-10-30T00:00:10.029,0.0,8.405426025390625,0,1,0,1,0,1,0 -2020-10-30T00:01:10.029,0.0,0.46718788146972656,0,1,0,1,0,1,0 -2020-10-30T00:02:10.029,0.0,0.4610404968261719,0,1,0,1,0,1,0 -2020-10-30T00:03:10.029,0.0,0.44185352325439453,0,1,0,1,0,1,0 -2020-10-30T00:04:10.029,16.0,0.04636049270629883,0,1,0,1,0,1,0 -2020-10-30T00:05:10.029,0.0,16.4781436920166,0,1,0,1,0,1,0 -2020-10-30T00:06:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:07:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:08:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:09:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:10:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:11:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:12:10.029,146.0,0.0,0,3,0,1,0,1,0 -2020-10-30T00:13:10.029,0.0,11.152569770812988,0,1,0,1,0,1,0 -2020-10-30T00:14:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:15:10.029,4.0,1.4813618659973145,0,1,0,1,0,1,0 -2020-10-30T00:16:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:17:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:18:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:19:10.029,0.0,5.509651184082031,0,1,0,1,0,1,0 -2020-10-30T00:20:10.029,8.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:21:10.029,0.0,7.602544784545898,0,1,0,1,0,1,0 -2020-10-30T00:22:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:23:10.029,5.5,0.0,0,1,0,1,0,1,0 -2020-10-30T00:24:10.029,48.0,5.560975074768066,0,1,0,1,0,1,0 -2020-10-30T00:25:10.029,0.0,25.713239669799805,0,1,0,1,0,1,0 -2020-10-30T00:26:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:27:10.029,103.0,0.0,0,2,0,1,0,1,0 -2020-10-30T00:28:10.029,5.0,37.42110061645508,0,1,0,1,0,1,0 -2020-10-30T00:29:10.029,0.0,0.2669506072998047,0,1,0,1,0,1,0 -2020-10-30T00:30:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:31:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:32:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:33:10.029,11.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:34:10.029,0.0,10.127628326416016,0,1,0,1,0,1,0 -2020-10-30T00:35:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:36:10.029,6.0,2.062410831451416,0,1,0,1,0,1,0 -2020-10-30T00:37:10.029,0.0,2.933445453643799,0,1,0,1,0,1,0 -2020-10-30T00:38:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:39:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:40:10.029,6.0,5.1230573654174805,0,1,0,1,0,1,0 -2020-10-30T00:41:10.029,5.0,4.622580528259277,0,1,0,1,0,1,0 -2020-10-30T00:42:10.029,0.0,3.64579439163208,0,1,0,1,0,1,0 -2020-10-30T00:43:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:44:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:45:10.029,0.0,8.640241622924805,0,1,0,1,0,1,0 -2020-10-30T00:46:10.029,88.0,0.6256647109985352,0,2,0,1,0,1,0 -2020-10-30T00:47:10.029,0.0,23.388309478759766,0,1,0,1,0,1,0 -2020-10-30T00:48:10.029,0.0,0.029178142547607422,0,1,0,1,0,1,0 -2020-10-30T00:49:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:50:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:51:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:52:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:53:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:54:10.029,0.0,6.692010879516602,0,1,0,1,0,1,0 -2020-10-30T00:55:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:56:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:57:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:58:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T00:59:10.029,6.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:00:10.029,4.0,4.232800483703613,0,1,0,1,0,1,0 -2020-10-30T01:01:10.029,0.0,1.832387924194336,0,1,0,1,0,1,0 -2020-10-30T01:02:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:03:10.029,0.0,4.666852951049805,0,1,0,1,0,1,0 -2020-10-30T01:04:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:05:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:06:10.029,16.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:07:10.029,0.0,11.525970458984375,0,1,0,1,0,1,0 -2020-10-30T01:08:10.029,5.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:09:10.029,0.0,6.69814395904541,0,1,0,1,0,1,0 -2020-10-30T01:10:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:11:10.029,7.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:12:10.029,0.0,9.639561653137207,0,1,0,1,0,1,0 -2020-10-30T01:13:10.029,6.0,0.04102945327758789,0,1,0,1,0,1,0 -2020-10-30T01:14:10.029,0.0,8.68164348602295,0,1,0,1,0,1,0 -2020-10-30T01:15:10.029,0.0,0.03974151611328125,0,1,0,1,0,1,0 -2020-10-30T01:16:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:17:10.029,4.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:18:10.029,0.0,7.123937606811523,0,1,0,1,0,1,0 -2020-10-30T01:19:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:20:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:21:10.029,0.0,0.0,0,1,0,1,0,1,0 -2020-10-30T01:22:10.029,193.0,0.0,1,4,0,1,0,1,0 -2020-10-30T01:23:10.029,0.0,20.622085571289062,1,1,0,1,0,1,0 -2020-10-30T01:24:10.029,67.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:25:10.029,0.0,22.343050003051758,1,1,0,1,0,1,0 -2020-10-30T01:26:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:27:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:28:10.029,0.0,1.5565133094787598,1,1,0,1,0,1,0 -2020-10-30T01:29:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:30:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:31:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:32:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:33:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:34:10.029,16.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:35:10.029,0.0,10.564291954040527,1,1,0,1,0,1,0 -2020-10-30T01:36:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:37:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:38:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:39:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:40:10.029,0.0,4.639544486999512,1,1,0,1,0,1,0 -2020-10-30T01:41:10.029,12.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:42:10.029,3.0,9.352090835571289,1,1,0,1,0,1,0 -2020-10-30T01:43:10.029,103.0,0.0,1,2,0,1,0,1,0 -2020-10-30T01:44:10.029,25.0,18.862722396850586,1,1,0,1,0,1,0 -2020-10-30T01:45:10.029,15.0,0.4382319450378418,1,1,0,1,0,1,0 -2020-10-30T01:46:10.029,9.0,3.2147507667541504,1,1,0,1,0,1,0 -2020-10-30T01:47:10.029,4.0,3.5339250564575195,1,1,0,1,0,1,0 -2020-10-30T01:48:10.029,12.0,2.22397518157959,1,1,0,1,0,1,0 -2020-10-30T01:49:10.029,6.0,12.766632080078125,1,1,0,1,0,1,0 -2020-10-30T01:50:10.029,22.0,4.93465518951416,1,1,0,1,0,1,0 -2020-10-30T01:51:10.029,7.0,30.895244598388672,1,1,0,1,0,1,0 -2020-10-30T01:52:10.029,0.0,13.804502487182617,1,1,0,1,0,1,0 -2020-10-30T01:53:10.029,0.0,6.213910102844238,1,1,0,1,0,1,0 -2020-10-30T01:54:10.029,0.0,3.929741382598877,1,1,0,1,0,1,0 -2020-10-30T01:55:10.029,0.0,1.8266377449035645,1,1,0,1,0,1,0 -2020-10-30T01:56:10.029,0.0,0.7386393547058105,1,1,0,1,0,1,0 -2020-10-30T01:57:10.029,0.0,0.08167219161987305,1,1,0,1,0,1,0 -2020-10-30T01:58:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T01:59:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:00:10.029,56.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:01:10.029,0.0,23.045989990234375,1,1,0,1,0,1,0 -2020-10-30T02:02:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:03:10.029,7.0,4.46674919128418,1,1,0,1,0,1,0 -2020-10-30T02:04:10.029,0.0,3.792720317840576,1,1,0,1,0,1,0 -2020-10-30T02:05:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:06:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:07:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:08:10.029,7.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:09:10.029,6.0,6.497780799865723,1,1,0,1,0,1,0 -2020-10-30T02:10:10.029,0.0,1.8255853652954102,1,1,0,1,0,1,0 -2020-10-30T02:11:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:12:10.029,13.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:13:10.029,5.0,12.702003479003906,1,1,0,1,0,1,0 -2020-10-30T02:14:10.029,0.0,1.0951943397521973,1,1,0,1,0,1,0 -2020-10-30T02:15:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:16:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:17:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:18:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:19:10.029,6.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:20:10.029,0.0,6.788580894470215,1,1,0,1,0,1,0 -2020-10-30T02:21:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:22:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:23:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:24:10.029,6.0,5.22747802734375,1,1,0,1,0,1,0 -2020-10-30T02:25:10.029,0.0,4.602972030639648,1,1,0,1,0,1,0 -2020-10-30T02:26:10.029,7.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:27:10.029,0.0,7.530411720275879,1,1,0,1,0,1,0 -2020-10-30T02:28:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:29:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:30:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:31:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:32:10.029,0.0,6.599978446960449,1,1,0,1,0,1,0 -2020-10-30T02:33:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:34:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:35:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:36:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:37:10.029,7.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:38:10.029,6.0,7.402435302734375,1,1,0,1,0,1,0 -2020-10-30T02:39:10.029,0.0,3.9300241470336914,1,1,0,1,0,1,0 -2020-10-30T02:40:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:41:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:42:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:43:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:44:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:45:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:46:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:47:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:48:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:49:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:50:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:51:10.029,148.0,0.0,1,3,0,1,0,1,0 -2020-10-30T02:52:10.029,4.0,47.786895751953125,1,1,0,1,0,1,0 -2020-10-30T02:53:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:54:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:55:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:56:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:57:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:58:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T02:59:10.029,7.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:00:10.029,0.0,4.276817321777344,1,1,0,1,0,1,0 -2020-10-30T03:01:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:02:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:03:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:04:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:05:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:06:10.029,7.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:07:10.029,0.0,5.683505058288574,1,1,0,1,0,1,0 -2020-10-30T03:08:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:09:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:10:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:11:10.029,6.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:12:10.029,0.0,6.073542594909668,1,1,0,1,0,1,0 -2020-10-30T03:13:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:14:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:15:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:16:10.029,6.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:17:10.029,0.0,6.23763370513916,1,1,0,1,0,1,0 -2020-10-30T03:18:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:19:10.029,21.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:20:10.029,0.0,9.620964050292969,1,1,0,1,0,1,0 -2020-10-30T03:21:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:22:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:23:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:24:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:25:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:26:10.029,4.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:27:10.029,0.0,4.076115608215332,1,1,0,1,0,1,0 -2020-10-30T03:28:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:29:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:30:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:31:10.029,0.0,4.418728828430176,1,1,0,1,0,1,0 -2020-10-30T03:32:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:33:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:34:10.029,7.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:35:10.029,0.0,7.036018371582031,1,1,0,1,0,1,0 -2020-10-30T03:36:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:37:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:38:10.029,131.0,0.0,1,3,0,1,0,1,0 -2020-10-30T03:39:10.029,198.0,18.23261260986328,1,4,0,1,0,1,0 -2020-10-30T03:40:10.029,0.0,25.32823944091797,1,1,0,1,0,1,0 -2020-10-30T03:41:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:42:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:43:10.029,4.0,1.189723014831543,1,1,0,1,0,1,0 -2020-10-30T03:44:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:45:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:46:10.029,0.0,1.6236763000488281,1,1,0,1,0,1,0 -2020-10-30T03:47:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:48:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:49:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:50:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:51:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:52:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:53:10.029,5.0,5.261150360107422,1,1,0,1,0,1,0 -2020-10-30T03:54:10.029,0.0,4.008538246154785,1,1,0,1,0,1,0 -2020-10-30T03:55:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:56:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:57:10.029,0.0,5.861323356628418,1,1,0,1,0,1,0 -2020-10-30T03:58:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T03:59:10.029,4.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:00:10.029,5.0,4.629955291748047,1,1,0,1,0,1,0 -2020-10-30T04:01:10.029,170.0,4.353062629699707,1,3,0,1,0,1,0 -2020-10-30T04:02:10.029,0.0,17.637008666992188,1,1,0,1,0,1,0 -2020-10-30T04:03:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:04:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:05:10.029,6.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:06:10.029,9.0,5.1784515380859375,1,1,0,1,0,1,0 -2020-10-30T04:07:10.029,6.0,4.843062400817871,1,1,0,1,0,1,0 -2020-10-30T04:08:10.029,5.0,1.1457085609436035,1,1,0,1,0,1,0 -2020-10-30T04:09:10.029,0.0,2.015923500061035,1,1,0,1,0,1,0 -2020-10-30T04:10:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:11:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:12:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:13:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:14:10.029,4.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:15:10.029,6.0,4.304697036743164,1,1,0,1,0,1,0 -2020-10-30T04:16:10.029,6.0,5.193272590637207,1,1,0,1,0,1,0 -2020-10-30T04:17:10.029,0.0,4.814790725708008,1,1,0,1,0,1,0 -2020-10-30T04:18:10.029,5.0,0.04473114013671875,1,1,0,1,0,1,0 -2020-10-30T04:19:10.029,0.0,8.80262565612793,1,1,0,1,0,1,0 -2020-10-30T04:20:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:21:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:22:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:23:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:24:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:25:10.029,5.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:26:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:27:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:28:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:29:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:30:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:31:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:32:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:33:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:34:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:35:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:36:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:37:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:38:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:39:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:40:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:41:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:42:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:43:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:44:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:45:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:46:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:47:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:48:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:49:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:50:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:51:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:52:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:53:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 -2020-10-30T04:54:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 -2020-10-30T04:55:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 -2020-10-30T04:56:10.029,0.0,5.873167991638184,1,1,0,1,0,1,0 -2020-10-30T04:57:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:58:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T04:59:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T05:00:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T05:01:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T05:02:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T05:03:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T05:04:10.029,0.0,0.0,1,1,0,1,0,1,0 -2020-10-30T05:05:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:06:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:07:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:08:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:09:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:10:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:11:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:12:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:13:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:14:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:15:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:16:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:17:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:18:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:19:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:20:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:21:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:22:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:23:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:24:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:25:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:26:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:27:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:28:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:29:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:30:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:31:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:32:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:33:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:34:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:35:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:36:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:37:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:38:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:39:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:40:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:41:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:42:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:43:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:44:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:45:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:46:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:47:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:48:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:49:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:50:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:51:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:52:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:53:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:54:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:55:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:56:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:57:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:58:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T05:59:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:00:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:01:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:02:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:03:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:04:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:05:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:06:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:07:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:08:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:09:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:10:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:11:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:12:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:13:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:14:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:15:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:16:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:17:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:18:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:19:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:20:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:21:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:22:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:23:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:24:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:25:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:26:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:27:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:28:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:29:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:30:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:31:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:32:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:33:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:34:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:35:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:36:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:37:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:38:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:39:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:40:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:41:10.029,0.0,0.0,-1,1,0,1,0,1,0 -2020-10-30T06:42:10.029,0.0,0.0,-1,1,0,1,0,1,0 diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py index 9576911..f5cc6ba 100644 --- a/FCRdataLoader/fcrdataloader/dataset.py +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -1,7 +1,7 @@ from torch.utils.data import Dataset, WeightedRandomSampler, DataLoader from pathlib import Path from typing import List, Callable -from abc import ABC, abstractmethod +from re import search import numpy as np import torch @@ -37,7 +37,7 @@ class SequenceForecastMultiDistributionDataset(Dataset): return self.getitem(self.start + idx) -class SequenceForecastMultiDistributionDatasetFactory(ABC): +class SequenceForecastMultiDistributionDatasetFactory: """ TODO update docs TODO add input validation @@ -74,7 +74,7 @@ class SequenceForecastMultiDistributionDatasetFactory(ABC): self, seq_len: int, pred_step: int, - files: List[str], + files: List[Path], x_y_split: int, usecols: List[int], x_predictions_cols: List[int], @@ -103,7 +103,7 @@ class SequenceForecastMultiDistributionDatasetFactory(ABC): self.x = [None] * len(files) self.y = [None] * len(files) - self.files = files + self.files = self.__sort_by_postfixnumb(files) self.transforms = transforms self.seq_len = seq_len self.x_y_split = x_y_split @@ -125,6 +125,13 @@ class SequenceForecastMultiDistributionDatasetFactory(ABC): ) return np.hsplit(series, [self.x_y_split]) + def __sort_by_postfixnumb(self, fnames: List[str]): + p = r'(\d+).' + return sorted( + fnames, + key=lambda x: int(search(p, x.name).groups()[-1]) + ) + def __preproces_data(self, seq_len, pred_step): sizes = np.zeros(len(self.files), dtype=np.int) for i, f_path in enumerate(self.files): @@ -248,16 +255,3 @@ class FCRdatasetFactory(SequenceForecastMultiDistributionDatasetFactory): usecols=range(1, 10), x_predictions_cols=range(1, 3) ) - - - -from torch.utils.data import DataLoader -from collections import Counter - - -FCRdsFactory = FCRdatasetFactory(115, 0) -train_set = FCRdsFactory.get_train_dataset() -loader = DataLoader(train_set, batch_size=len(train_set), sampler=FCRdsFactory.get_weighted_rnd_sampler()) - -cnt = Counter(map(lambda x: tuple(x.numpy()), next(iter(loader))[1])) -print(cnt.values()) diff --git a/FCRdataLoader/setup.py b/FCRdataLoader/setup.py index 23a3360..5c3d74d 100644 --- a/FCRdataLoader/setup.py +++ b/FCRdataLoader/setup.py @@ -153,4 +153,5 @@ setup( # issues, where the source is hosted, where to say thanks to the package # maintainers, and where to support the project financially. The key is # what's used to render the link text on PyPI. + test_suite="tests", ) diff --git a/FCRdataLoader/tests/__init__.py b/FCRdataLoader/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FCRdataLoader/tests/seq1pred0_test.py b/FCRdataLoader/tests/seq1pred0_test.py new file mode 100644 index 0000000..9617f12 --- /dev/null +++ b/FCRdataLoader/tests/seq1pred0_test.py @@ -0,0 +1,35 @@ +import unittest +from fcrdataloader.dataset import SequenceForecastMultiDistributionDatasetFactory as SFMDDF +from torch.utils.data import DataLoader +from pathlib import Path +import os + + +TEST_FILE_DIR = 'tests/test_data' + + +class FCRCorrectDataTest(unittest.TestCase): + + def test_seq1_pred0(self): + factory = SFMDDF( + seq_len=1, + pred_step=0, + files=list(Path(TEST_FILE_DIR).glob("*")), + x_y_split=3, + usecols=range(1, 10), + x_predictions_cols=range(1, 3) + ) + train_set = factory.get_train_dataset() + loader = DataLoader(train_set, batch_size=1) + + for row, (x, y) in enumerate(loader): + x = x.reshape(-1) + y = y.reshape(-1) + for col in range(0, 3): + self.assertEqual(x[col], float(f"{row}.{col}")) + for col in range(0, 6): + self.assertEqual(y[col], float(f"{row}.{col + 3}")) + + +if __name__ == '__main__': + unittest.main() diff --git a/FCRdataLoader/tests/test_data/fcrdata10.csv b/FCRdataLoader/tests/test_data/fcrdata10.csv new file mode 100644 index 0000000..285e8c5 --- /dev/null +++ b/FCRdataLoader/tests/test_data/fcrdata10.csv @@ -0,0 +1,12 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-11-27T12:44:35.519,10.0,10.1,10.2,10.3,10.4,10.5,10.6,10.7,10.8,10.9 +2020-11-27T12:44:35.519,11.0,11.1,11.2,11.3,11.4,11.5,11.6,11.7,11.8,11.9 +2020-11-27T12:44:35.519,12.0,12.1,12.2,12.3,12.4,12.5,12.6,12.7,12.8,12.9 +2020-11-27T12:44:35.519,13.0,13.1,13.2,13.3,13.4,13.5,13.6,13.7,13.8,13.9 +2020-11-27T12:44:35.519,14.0,14.1,14.2,14.3,14.4,14.5,14.6,14.7,14.8,14.9 +2020-11-27T12:44:35.519,15.0,15.1,15.2,15.3,15.4,15.5,15.6,15.7,15.8,15.9 +2020-11-27T12:44:35.519,16.0,16.1,16.2,16.3,16.4,16.5,16.6,16.7,16.8,16.9 +2020-11-27T12:44:35.519,17.0,17.1,17.2,17.3,17.4,17.5,17.6,17.7,17.8,17.9 +2020-11-27T12:44:35.519,18.0,18.1,18.2,18.3,18.4,18.5,18.6,18.7,18.8,18.9 +2020-11-27T12:44:35.519,18.0,18.1,18.2,18.3,18.4,18.5,18.6,18.7,18.8,18.9 +2020-11-27T12:44:35.519,20.0,20.1,20.2,20.3,20.4,20.5,20.6,20.7,20.8,20.9 diff --git a/FCRdataLoader/tests/test_data/fcrdata2.csv b/FCRdataLoader/tests/test_data/fcrdata2.csv new file mode 100644 index 0000000..7ef612e --- /dev/null +++ b/FCRdataLoader/tests/test_data/fcrdata2.csv @@ -0,0 +1,11 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-11-27T12:44:35.519,0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9 +2020-11-27T12:44:35.519,1.0,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8,1.9 +2020-11-27T12:44:35.519,2.0,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8,2.9 +2020-11-27T12:44:35.519,3.0,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8,3.9 +2020-11-27T12:44:35.519,4.0,4.1,4.2,4.3,4.4,4.5,4.6,4.7,4.8,4.9 +2020-11-27T12:44:35.519,5.0,5.1,5.2,5.3,5.4,5.5,5.6,5.7,5.8,5.9 +2020-11-27T12:44:35.519,6.0,6.1,6.2,6.3,6.4,6.5,6.6,6.7,6.8,6.9 +2020-11-27T12:44:35.519,7.0,7.1,7.2,7.3,7.4,7.5,7.6,7.7,7.8,7.9 +2020-11-27T12:44:35.519,8.0,8.1,8.2,8.3,8.4,8.5,8.6,8.7,8.8,8.9 +2020-11-27T12:44:35.519,9.0,9.1,9.2,9.3,9.4,9.5,9.6,9.7,9.8,9.9 -- GitLab From a9b8341c4d0b969d8ae7878f96a89738985d9107 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Mon, 18 Jan 2021 11:52:37 +0100 Subject: [PATCH 17/26] updeted readme --- FCRdataLoader/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/FCRdataLoader/README.md b/FCRdataLoader/README.md index b10bd50..5e16c6c 100644 --- a/FCRdataLoader/README.md +++ b/FCRdataLoader/README.md @@ -21,6 +21,12 @@ $ pip install -r requirements.txt - containing test data ### details about pytorch dataset and DataLoader can be found here: https://pytorch.org/docs/stable/data.html +# Tests + - to run automatic tests run +```sh +$ python setup.py test +``` +

# Example of usage -- GitLab From 2c213d57d5bce8c551786050b8d523caed59cdb0 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Mon, 25 Jan 2021 19:26:49 +0100 Subject: [PATCH 18/26] dataset updates --- FCRdataLoader/fcrdataloader/dataset.py | 15 ++++-- FCRtraining/networks/lstm_network1.py | 70 +++++++++++++++++++------- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/fcrdataloader/dataset.py index f5cc6ba..c27997b 100644 --- a/FCRdataLoader/fcrdataloader/dataset.py +++ b/FCRdataLoader/fcrdataloader/dataset.py @@ -182,7 +182,10 @@ class SequenceForecastMultiDistributionDatasetFactory: return new_cols - def get_weighted_rnd_sampler(self) -> WeightedRandomSampler: + def set_transforms(self, transforms): + self.transforms = transforms + + def get_uniform_dist_y_sampler(self) -> WeightedRandomSampler: """ returns instance of torch.utils.data.WeightedRandomSampler with weights so that after applying it to as sampler in @@ -191,7 +194,7 @@ class SequenceForecastMultiDistributionDatasetFactory: """ train_ds = self.get_train_dataset() - loader = DataLoader(train_set, batch_size=len(train_ds)) + loader = DataLoader(train_ds, batch_size=len(train_ds)) unique, indexes, occurs = np.unique( next(iter(loader))[1], # get all "y" from train_dataset (outputs) return_counts=True, @@ -229,8 +232,11 @@ class SequenceForecastMultiDistributionDatasetFactory: def get_train_dataset(self) -> SequenceForecastMultiDistributionDataset: return self.__get_dataset(0, self.__train_size(self.size)) - def get_test_set(self) -> SequenceForecastMultiDistributionDataset: - return self.__get_dataset(self.__test_size(self.size), self.size) + def get_test_dataset(self) -> SequenceForecastMultiDistributionDataset: + return self.__get_dataset( + self.__train_size(self.size), + self.__test_size(self.size) + ) class FCRdatasetFactory(SequenceForecastMultiDistributionDatasetFactory): @@ -255,3 +261,4 @@ class FCRdatasetFactory(SequenceForecastMultiDistributionDatasetFactory): usecols=range(1, 10), x_predictions_cols=range(1, 3) ) + diff --git a/FCRtraining/networks/lstm_network1.py b/FCRtraining/networks/lstm_network1.py index 943db17..9b4eb18 100644 --- a/FCRtraining/networks/lstm_network1.py +++ b/FCRtraining/networks/lstm_network1.py @@ -1,16 +1,24 @@ -from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet, FCRtestDataSet +from FCRdataLoader.fcrdataloader.dataset import FCRdatasetFactory from torch.utils.data import DataLoader from torch.optim.lr_scheduler import ReduceLROnPlateau +from sklearn.preprocessing import MinMaxScaler, StandardScaler from .LitFCRtestBase import BaseTestEncoder +import torch.nn.functional as F import torch.nn as nn import torch -HIDDEN_SIZE = 40 -BATCH_SIZE = 32 -SEQ_LEN = 10 -HORIZON = 0 -LR = 0.01 +''' + Dont touch great performance +''' + + +HIDDEN_SIZE = 30 +BATCH_SIZE = 128 +SEQ_LEN = 20 +HORIZON = 5 +LSTM_LAYERS = 3 +LR = 0.1 FEATURES = 3 OUTPUT = 6 @@ -21,11 +29,12 @@ class Encoder(BaseTestEncoder): self, features=FEATURES, output=OUTPUT, - learning_rate=LR, + lr=LR, batch_size=BATCH_SIZE, seq_len=SEQ_LEN, horizon=HORIZON, hidden_size=HIDDEN_SIZE, + lstm_layers=LSTM_LAYERS ): super(Encoder, self).__init__() @@ -33,25 +42,32 @@ class Encoder(BaseTestEncoder): self.seq_len = seq_len self.horizon = horizon self.batch_size = batch_size + self.lstm_layers = LSTM_LAYERS self.criterion = nn.MSELoss() - self.lr = learning_rate + self.lr = lr - self.lstm = nn.LSTM(features, hidden_size, num_layers=2, + self.lstm = nn.LSTM(features, hidden_size, num_layers=self.lstm_layers, bidirectional=True, batch_first=True) - self.fc = nn.Linear(hidden_size * 2, output) + self.fc1 = nn.Linear(hidden_size * 2, output) + + # data transformation + self.data_set_factory = FCRdatasetFactory(SEQ_LEN, HORIZON) def forward(self, x): out, _ = self.lstm(x) # out: (batch, features, hidden_size * directions) out = out[:, -1, :] # out: (batch, hidden_size * directions) - out = self.fc(out) + out = self.fc1(out) return out def training_step(self, batch, batch_idx): x, y = batch prediction = self(x) + #print(f"x = {x[0]}") + #print(f"pred = {torch.round(prediction[0])}") + #print(f"y = {y[0]}") loss = self.criterion(prediction, y) self.log('train_loss', loss, on_step=False, on_epoch=True) @@ -62,23 +78,39 @@ class Encoder(BaseTestEncoder): return self.test_dataloader() def train_dataloader(self): - train_data = FCRtrainDataSet(self.seq_len, self.horizon) - loader = DataLoader(train_data, batch_size=self.batch_size, - num_workers=4) - return loader + return DataLoader( + self.data_set_factory.get_train_dataset(), + batch_size=self.batch_size, + num_workers=4, + sampler=self.data_set_factory.get_uniform_dist_y_sampler() + ) def test_dataloader(self): - test_data = FCRtestDataSet(self.seq_len, self.horizon) - loader = DataLoader(test_data, batch_size=self.batch_size, - num_workers=4) + loader = DataLoader( + self.data_set_factory.get_test_dataset(), + batch_size=self.batch_size, + num_workers=4 + ) return loader def configure_optimizers(self): optimizer = torch.optim.Adam(self.parameters(), lr=self.lr) scheduler = ReduceLROnPlateau( - optimizer, 'min', patience=10, verbose=True) + optimizer, 'min', patience=20, verbose=True) return { 'optimizer': optimizer, 'lr_scheduler': scheduler, 'monitor': 'train_loss' } + + def __get_scaler(self, train_dataset): + scaler_loader = DataLoader( + train_dataset, + batch_size=len(train_dataset) + ) + + scaler = MinMaxScaler(feature_range=(-10, 10)) + batch = next(iter(scaler_loader))[0].reshape(-1, 3) # fixed for fcr data + scaler.fit(batch) + + return scaler -- GitLab From 2d957a8908db878fc59cfc64f97a230bbd679db0 Mon Sep 17 00:00:00 2001 From: szymon sadkowski Date: Mon, 25 Jan 2021 19:27:25 +0100 Subject: [PATCH 19/26] removed private file --- FCRtraining/networks/lstm_network1.py | 116 -------------------------- 1 file changed, 116 deletions(-) delete mode 100644 FCRtraining/networks/lstm_network1.py diff --git a/FCRtraining/networks/lstm_network1.py b/FCRtraining/networks/lstm_network1.py deleted file mode 100644 index 9b4eb18..0000000 --- a/FCRtraining/networks/lstm_network1.py +++ /dev/null @@ -1,116 +0,0 @@ -from FCRdataLoader.fcrdataloader.dataset import FCRdatasetFactory -from torch.utils.data import DataLoader -from torch.optim.lr_scheduler import ReduceLROnPlateau -from sklearn.preprocessing import MinMaxScaler, StandardScaler -from .LitFCRtestBase import BaseTestEncoder -import torch.nn.functional as F -import torch.nn as nn -import torch - - -''' - Dont touch great performance -''' - - -HIDDEN_SIZE = 30 -BATCH_SIZE = 128 -SEQ_LEN = 20 -HORIZON = 5 -LSTM_LAYERS = 3 -LR = 0.1 - -FEATURES = 3 -OUTPUT = 6 - - -class Encoder(BaseTestEncoder): - def __init__( - self, - features=FEATURES, - output=OUTPUT, - lr=LR, - batch_size=BATCH_SIZE, - seq_len=SEQ_LEN, - horizon=HORIZON, - hidden_size=HIDDEN_SIZE, - lstm_layers=LSTM_LAYERS - - ): - super(Encoder, self).__init__() - - self.seq_len = seq_len - self.horizon = horizon - self.batch_size = batch_size - self.lstm_layers = LSTM_LAYERS - - self.criterion = nn.MSELoss() - self.lr = lr - - self.lstm = nn.LSTM(features, hidden_size, num_layers=self.lstm_layers, - bidirectional=True, batch_first=True) - self.fc1 = nn.Linear(hidden_size * 2, output) - - # data transformation - self.data_set_factory = FCRdatasetFactory(SEQ_LEN, HORIZON) - - def forward(self, x): - out, _ = self.lstm(x) - # out: (batch, features, hidden_size * directions) - out = out[:, -1, :] - # out: (batch, hidden_size * directions) - out = self.fc1(out) - return out - - def training_step(self, batch, batch_idx): - x, y = batch - prediction = self(x) - #print(f"x = {x[0]}") - #print(f"pred = {torch.round(prediction[0])}") - #print(f"y = {y[0]}") - loss = self.criterion(prediction, y) - - self.log('train_loss', loss, on_step=False, on_epoch=True) - - return loss - - def val_dataloader(self): - return self.test_dataloader() - - def train_dataloader(self): - return DataLoader( - self.data_set_factory.get_train_dataset(), - batch_size=self.batch_size, - num_workers=4, - sampler=self.data_set_factory.get_uniform_dist_y_sampler() - ) - - def test_dataloader(self): - loader = DataLoader( - self.data_set_factory.get_test_dataset(), - batch_size=self.batch_size, - num_workers=4 - ) - return loader - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.parameters(), lr=self.lr) - scheduler = ReduceLROnPlateau( - optimizer, 'min', patience=20, verbose=True) - return { - 'optimizer': optimizer, - 'lr_scheduler': scheduler, - 'monitor': 'train_loss' - } - - def __get_scaler(self, train_dataset): - scaler_loader = DataLoader( - train_dataset, - batch_size=len(train_dataset) - ) - - scaler = MinMaxScaler(feature_range=(-10, 10)) - batch = next(iter(scaler_loader))[0].reshape(-1, 3) # fixed for fcr data - scaler.fit(batch) - - return scaler -- GitLab From 14fb4df0dc3398bc9a755aa5c8a619fb4fb64a95 Mon Sep 17 00:00:00 2001 From: jkk Date: Tue, 2 Feb 2021 21:37:46 +0100 Subject: [PATCH 20/26] Package reformat + improved setuptools script --- .../{ => src}/fcrdataloader/__init__.py | 0 FCRdataLoader/{ => src}/fcrdataloader/data.py | 0 .../{ => src}/fcrdataloader/dataset.py | 0 FCRgendata/setup.py | 39 ++------- FCRgendata/src/{ => FCRGenData}/__init__.py | 0 FCRgendata/src/FCRGenData/__main__.py | 82 +++++++++++++++++++ .../src/{ => FCRGenData}/solv_summoner.py | 0 .../src/{ => FCRGenData}/validate_config.py | 0 FCRgendata/src/main.py | 78 ------------------ 9 files changed, 89 insertions(+), 110 deletions(-) rename FCRdataLoader/{ => src}/fcrdataloader/__init__.py (100%) rename FCRdataLoader/{ => src}/fcrdataloader/data.py (100%) rename FCRdataLoader/{ => src}/fcrdataloader/dataset.py (100%) rename FCRgendata/src/{ => FCRGenData}/__init__.py (100%) create mode 100644 FCRgendata/src/FCRGenData/__main__.py rename FCRgendata/src/{ => FCRGenData}/solv_summoner.py (100%) rename FCRgendata/src/{ => FCRGenData}/validate_config.py (100%) delete mode 100644 FCRgendata/src/main.py diff --git a/FCRdataLoader/fcrdataloader/__init__.py b/FCRdataLoader/src/fcrdataloader/__init__.py similarity index 100% rename from FCRdataLoader/fcrdataloader/__init__.py rename to FCRdataLoader/src/fcrdataloader/__init__.py diff --git a/FCRdataLoader/fcrdataloader/data.py b/FCRdataLoader/src/fcrdataloader/data.py similarity index 100% rename from FCRdataLoader/fcrdataloader/data.py rename to FCRdataLoader/src/fcrdataloader/data.py diff --git a/FCRdataLoader/fcrdataloader/dataset.py b/FCRdataLoader/src/fcrdataloader/dataset.py similarity index 100% rename from FCRdataLoader/fcrdataloader/dataset.py rename to FCRdataLoader/src/fcrdataloader/dataset.py diff --git a/FCRgendata/setup.py b/FCRgendata/setup.py index ad61eca..30f974b 100644 --- a/FCRgendata/setup.py +++ b/FCRgendata/setup.py @@ -187,36 +187,11 @@ setup( # see https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi # dependency_links=[], - # If using Python 2.6 or earlier, then these have to be included in - # MANIFEST.in as well. - # package_data={"sample": ["package_data.dat"]}, # Optional - # Although 'package_data' is the preferred approach, in some case you may - # need to place data files outside of your packages. See: - # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files - # - # In this case, 'data_file' will be installed into '/my_data' - # data_files=[("my_data", ["data/data_file"])], # Optional - # To provide executable scripts, use entry points in preference to the - # "scripts" keyword. Entry points provide cross-platform support and allow - # `pip` to create the appropriate form of executable for the target - # platform. - # - # For example, the following would provide a command called `sample` which - # executes the function `main` from this package when invoked: - # entry_points={"console_scripts": ["sample=sample:main"]}, # Optional - # List additional URLs that are relevant to your project as a dict. - # - # This field corresponds to the "Project-URL" metadata fields: - # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use - # - # Examples listed include a pattern for specifying where the package tracks - # issues, where the source is hosted, where to say thanks to the package - # maintainers, and where to support the project financially. The key is - # what's used to render the link text on PyPI. - #project_urls={ # Optional - # "Bug Reports": "https://github.com/pypa/sampleproject/issues", - # "Funding": "https://donate.pypi.org", - # "Say Thanks!": "http://saythanks.io/to/example", - # "Source": "https://github.com/pypa/sampleproject/", - #}, + package_dir={'': 'src'}, + entry_points={ + 'console_scripts': [ + 'fcrgendata = FCRGenData.__main__:run', + ], + }, + ) diff --git a/FCRgendata/src/__init__.py b/FCRgendata/src/FCRGenData/__init__.py similarity index 100% rename from FCRgendata/src/__init__.py rename to FCRgendata/src/FCRGenData/__init__.py diff --git a/FCRgendata/src/FCRGenData/__main__.py b/FCRgendata/src/FCRGenData/__main__.py new file mode 100644 index 0000000..a81e509 --- /dev/null +++ b/FCRgendata/src/FCRGenData/__main__.py @@ -0,0 +1,82 @@ +""" Generates data using CP-solver and FCR time series for network training """ + +import csv +import logging +import sys +from contextlib import ExitStack +from pathlib import Path + +import numpy as np +import numpy.lib.recfunctions as rfn +from progress.bar import IncrementalBar as IBar + +from .solv_summoner import CPSolverSolutionSummoner, CPSolverSolutionSummonerError +from .validate_config import validate_config + +logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) +CONFIG_PATH: Path = None + + +def run(): + if len(sys.argv) != 2: + logging.critical( + f"received {len(sys.argv) - 1} arguments, required one: config file path.") + sys.exit(1) + else: + CONFIG_PATH = Path(sys.argv[1]) + + j_conf = validate_config(CONFIG_PATH) + avg_rsp_times = np.genfromtxt( # shape (n, ) + j_conf['AvgResponseTimeTableFilePath'], + delimiter=',', + dtype=np.dtype([('avg_rsp_time', np.float64), ('datetime', 'datetime64[s]')]), + skip_header=1, + usecols=(6, 1) + ) + + avg_rsp_prediction = np.genfromtxt( # shape (n, ) + j_conf['predictionsFilePath'], + delimiter=',', + dtype=np.dtype([('avg_rsp_time_pred', np.float64), ('datetime', 'datetime64[s]'), ('split', np.int)]), + skip_header=1, + usecols=(1, 3, 4), + converters={4: lambda split: {b"train": 0, b"val": 1, b"test": -1}[split]} + ) + + joined = rfn.join_by('datetime', avg_rsp_times, avg_rsp_prediction, jointype='inner', usemask=False) + assert len(joined) == len(avg_rsp_prediction) + # joined is structured array of tuples with dtype + # [('datetime', ' Date: Fri, 12 Feb 2021 21:58:22 +0100 Subject: [PATCH 21/26] Data reader interface prototype --- FCRgendata/src/FCRGenData/raw_data_reader.py | 25 ++++++++++++++++++++ FCRgendata/tests/test_data_reader.py | 7 ++++++ 2 files changed, 32 insertions(+) create mode 100644 FCRgendata/src/FCRGenData/raw_data_reader.py create mode 100644 FCRgendata/tests/test_data_reader.py diff --git a/FCRgendata/src/FCRGenData/raw_data_reader.py b/FCRgendata/src/FCRGenData/raw_data_reader.py new file mode 100644 index 0000000..0f5f841 --- /dev/null +++ b/FCRgendata/src/FCRGenData/raw_data_reader.py @@ -0,0 +1,25 @@ +import abc +from typing import Tuple, Dict, Generator, List, Iterable + + +class IRawDataProvider(abc.ABC): + + + @abc.abstractmethod + @property + def column_names(self) -> Tuple[str]: + pass + + @abc.abstractmethod + @property + def columns(self) -> Dict[str, type]: + pass + + @abc.abstractmethod + def reader(self) -> Generator[Iterable[type], None, None]: + pass + + @abc.abstractmethod + def reader_annotated(self) -> Generator[Dict[str, type], None, None]: + pass + diff --git a/FCRgendata/tests/test_data_reader.py b/FCRgendata/tests/test_data_reader.py new file mode 100644 index 0000000..d7dd395 --- /dev/null +++ b/FCRgendata/tests/test_data_reader.py @@ -0,0 +1,7 @@ +import abc +import pytest + + +class RawDataProviderTestTemplate(abc.ABC): + pass + # TODO tests common for each implementation \ No newline at end of file -- GitLab From d25c265040e33354d7a631a83b14a4c31a0423c2 Mon Sep 17 00:00:00 2001 From: jkk Date: Fri, 12 Feb 2021 22:24:33 +0100 Subject: [PATCH 22/26] Added comments --- FCRgendata/src/FCRGenData/raw_data_reader.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/FCRgendata/src/FCRGenData/raw_data_reader.py b/FCRgendata/src/FCRGenData/raw_data_reader.py index 0f5f841..23d0377 100644 --- a/FCRgendata/src/FCRGenData/raw_data_reader.py +++ b/FCRgendata/src/FCRGenData/raw_data_reader.py @@ -1,25 +1,35 @@ import abc -from typing import Tuple, Dict, Generator, List, Iterable +from typing import Tuple, Dict, Generator, Iterable class IRawDataProvider(abc.ABC): - + """Data provider interface specification. Supplied with + time-series data generators, which provides data measured in + increasing, non repetitive timestamps. Time between timestamps *may* + vary.""" @abc.abstractmethod @property def column_names(self) -> Tuple[str]: + """Names of the variable columns""" pass @abc.abstractmethod @property def columns(self) -> Dict[str, type]: + """Names of the variable columns with mapping to it's types""" pass @abc.abstractmethod def reader(self) -> Generator[Iterable[type], None, None]: + """Generator over raw data, provides rows of data in increasing order (by timestamp) + Returns list with values (order same as in column_names). + """ pass @abc.abstractmethod def reader_annotated(self) -> Generator[Dict[str, type], None, None]: + """Generator over raw data, provides rows of data in increasing order (by timestamp). + Returns dict with column names mapping to current values. + """ pass - -- GitLab From 127670cfb6b7226770899f28cbcff855afa34e5d Mon Sep 17 00:00:00 2001 From: jkk <32602977+j-kk@users.noreply.github.com> Date: Wed, 17 Feb 2021 22:43:03 +0100 Subject: [PATCH 23/26] CSV reader basic implementation --- .../src/FCRGenData/rawDataReader/__init__.py | 2 + .../FCRGenData/rawDataReader/csv_reader.py | 94 +++++++++++++++++++ .../{ => rawDataReader}/raw_data_reader.py | 0 3 files changed, 96 insertions(+) create mode 100644 FCRgendata/src/FCRGenData/rawDataReader/__init__.py create mode 100644 FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py rename FCRgendata/src/FCRGenData/{ => rawDataReader}/raw_data_reader.py (100%) diff --git a/FCRgendata/src/FCRGenData/rawDataReader/__init__.py b/FCRgendata/src/FCRGenData/rawDataReader/__init__.py new file mode 100644 index 0000000..60afb7f --- /dev/null +++ b/FCRgendata/src/FCRGenData/rawDataReader/__init__.py @@ -0,0 +1,2 @@ +from .raw_data_reader import IRawDataProvider +from .csv_reader import RawCSVReader \ No newline at end of file diff --git a/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py b/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py new file mode 100644 index 0000000..0196b67 --- /dev/null +++ b/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py @@ -0,0 +1,94 @@ +import csv +import datetime as dt +import logging +from pathlib import Path +from typing import Generator, Dict, Iterable, Tuple, Union, List, Optional + +import pandas as pd + +from .raw_data_reader import IRawDataProvider + +logger = logging.getLogger(__name__) + + +class RawCSVReader(IRawDataProvider): + __path: Path + __delimiter: str + + __arr: pd.DataFrame + + __lines: List[str] + + __column_names: Tuple[str] + __column_types: Dict[str, type] + + def __init__(self, + path: Union[Path, str], + delimiter: str = ',', + timestamp_column_name: Optional[str] = None + ): + assert path is not None, 'Unset path!' + if isinstance(path, str): + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f'File {path} not found') + self.__path = path + self.__delimiter = delimiter + + # Peek headers + with open(self.__path, newline='') as file: + assert csv.Sniffer().has_header(file.readline()), "CSV file has no header" + file.seek(0) + columns = file.readline().strip('\n').split(delimiter) + + time_columns = list(filter(lambda name: 'time' in name or 'timestamp' in name, columns)) + + if timestamp_column_name == None: + timestamp_columns = list(filter(lambda name: 'datetime' in name or 'timestamp' in name, columns)) + else: + timestamp_columns = list(filter(lambda name: timestamp_column_name in name, columns)) + + assert len(timestamp_columns) == 1, f'Cannot specify timestamp column, found column names: {columns}' + timestamp_column_name = timestamp_columns[0] + + # Read from file = be aware of high memory usage + self.__arr = pd.read_csv( + path=path, + parse_dates=time_columns + ) + assert self.__arr[timestamp_column_name].is_monotonic_increasing, 'Timestamps in column are not increasing' + + for key, name in self.__arr.apply(lambda x: x.name).to_dict().items(): + if name == 'object': + new_value = str + elif name.startswith('datetime'): + new_value = dt.datetime + elif name.startswith('float'): + new_value = float + elif name.startswith('int'): + new_value = int + elif name == 'bool': + new_value = bool + else: + raise NotImplementedError("Unknown datatype format") + self.__column_types[key] = new_value + + @property + def column_names(self) -> Tuple[str]: + return tuple(self.__arr.columns) + + @property + def columns(self) -> Dict[str, type]: + return self.__column_types + + def reader(self) -> Generator[Iterable[type], None, None]: + for index, row in self.__arr.iterrows(): + yield row.values.tolist() + + def reader_annotated(self) -> Generator[Dict[str, type], None, None]: + for index, row in self.__arr.iterrows(): + mapped_values = dict(row.to_dict()) + for key, val in mapped_values.items(): + if isinstance(val, pd._libs.tslibs.timestamps.Timestamp): + mapped_values[key] = val.to_pydatetime() + yield mapped_values diff --git a/FCRgendata/src/FCRGenData/raw_data_reader.py b/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py similarity index 100% rename from FCRgendata/src/FCRGenData/raw_data_reader.py rename to FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py -- GitLab From 73e8225f56917f923295194401b77e00cc3e6bf2 Mon Sep 17 00:00:00 2001 From: jkk <32602977+j-kk@users.noreply.github.com> Date: Wed, 17 Feb 2021 22:57:26 +0100 Subject: [PATCH 24/26] Added comments --- .../FCRGenData/rawDataReader/csv_reader.py | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py b/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py index 0196b67..3025179 100644 --- a/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py +++ b/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py @@ -10,8 +10,17 @@ from .raw_data_reader import IRawDataProvider logger = logging.getLogger(__name__) +TIME_COLUMN_NAMES = ['time', 'date', 'timestamp', 'datetime'] +TIMESTAMP_KEY_COLUMN_NAMES = ['timestamp', 'datetime'] + + +def _match_columns(column: List[str], keys: List[str]) -> List[str]: + """Filters column with names including any phrase from keys list.""" + return list(filter(lambda itername: any(map(lambda keyname: keyname in itername, keys)), column)) + class RawCSVReader(IRawDataProvider): + """CSV data reader implementation.""" __path: Path __delimiter: str @@ -27,6 +36,13 @@ class RawCSVReader(IRawDataProvider): delimiter: str = ',', timestamp_column_name: Optional[str] = None ): + """ + Args: + path: (Union[Path, str]) path to the csv file. + delimiter: (str) csv file felimiter. + timestamp_column_name: (str) timestamp column name (if unset, it will be guessed). + """ + # Check args assert path is not None, 'Unset path!' if isinstance(path, str): path = Path(path) @@ -41,12 +57,12 @@ class RawCSVReader(IRawDataProvider): file.seek(0) columns = file.readline().strip('\n').split(delimiter) - time_columns = list(filter(lambda name: 'time' in name or 'timestamp' in name, columns)) + time_columns = _match_columns(columns, TIME_COLUMN_NAMES) - if timestamp_column_name == None: - timestamp_columns = list(filter(lambda name: 'datetime' in name or 'timestamp' in name, columns)) + if timestamp_column_name is None: + timestamp_columns = _match_columns(columns, TIMESTAMP_KEY_COLUMN_NAMES) else: - timestamp_columns = list(filter(lambda name: timestamp_column_name in name, columns)) + timestamp_columns = _match_columns(columns, [timestamp_column_name]) assert len(timestamp_columns) == 1, f'Cannot specify timestamp column, found column names: {columns}' timestamp_column_name = timestamp_columns[0] @@ -75,17 +91,29 @@ class RawCSVReader(IRawDataProvider): @property def column_names(self) -> Tuple[str]: + """Column names""" return tuple(self.__arr.columns) @property def columns(self) -> Dict[str, type]: + """Column names mapping to it's types""" return self.__column_types def reader(self) -> Generator[Iterable[type], None, None]: + """Returns iterator over rows. + + Yields: + Iterable[type]: values in the next row (order is the same as in column_names). + """ for index, row in self.__arr.iterrows(): yield row.values.tolist() def reader_annotated(self) -> Generator[Dict[str, type], None, None]: + """Returns dict iterator over rows. + + Yields: + Dicttype[str, ]: name of the columns mapping to its values in the current row. + """ for index, row in self.__arr.iterrows(): mapped_values = dict(row.to_dict()) for key, val in mapped_values.items(): -- GitLab From 3c8d40b1678e56df4243f88e998c24f728bbf236 Mon Sep 17 00:00:00 2001 From: jkk Date: Fri, 19 Feb 2021 00:46:00 +0100 Subject: [PATCH 25/26] Abstract property decorator fixes --- FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py b/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py index 23d0377..43c1313 100644 --- a/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py +++ b/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py @@ -8,14 +8,14 @@ class IRawDataProvider(abc.ABC): increasing, non repetitive timestamps. Time between timestamps *may* vary.""" - @abc.abstractmethod @property + @abc.abstractmethod def column_names(self) -> Tuple[str]: """Names of the variable columns""" pass - @abc.abstractmethod @property + @abc.abstractmethod def columns(self) -> Dict[str, type]: """Names of the variable columns with mapping to it's types""" pass -- GitLab From 2671465f91f90b1177f6a547f12a5a88fd25cd7f Mon Sep 17 00:00:00 2001 From: jkk Date: Fri, 19 Feb 2021 15:14:28 +0100 Subject: [PATCH 26/26] Added tests & bug fixes --- .../FCRGenData/rawDataReader/csv_reader.py | 36 ++--- .../rawDataReader/raw_data_reader.py | 14 +- .../tests/data_reader/data_reader_template.py | 124 ++++++++++++++++++ .../tests/data_reader/test_csv_reader.py | 28 ++++ FCRgendata/tests/test_data_reader.py | 7 - 5 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 FCRgendata/tests/data_reader/data_reader_template.py create mode 100644 FCRgendata/tests/data_reader/test_csv_reader.py delete mode 100644 FCRgendata/tests/test_data_reader.py diff --git a/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py b/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py index 3025179..20fe224 100644 --- a/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py +++ b/FCRgendata/src/FCRGenData/rawDataReader/csv_reader.py @@ -6,13 +6,10 @@ from typing import Generator, Dict, Iterable, Tuple, Union, List, Optional import pandas as pd -from .raw_data_reader import IRawDataProvider +from .raw_data_reader import IRawDataProvider, TIME_COLUMN_NAMES logger = logging.getLogger(__name__) -TIME_COLUMN_NAMES = ['time', 'date', 'timestamp', 'datetime'] -TIMESTAMP_KEY_COLUMN_NAMES = ['timestamp', 'datetime'] - def _match_columns(column: List[str], keys: List[str]) -> List[str]: """Filters column with names including any phrase from keys list.""" @@ -34,8 +31,9 @@ class RawCSVReader(IRawDataProvider): def __init__(self, path: Union[Path, str], delimiter: str = ',', - timestamp_column_name: Optional[str] = None - ): + timestamp_column_name: Optional[str] = None): + super().__init__(timestamp_column_name=timestamp_column_name) + """ Args: path: (Union[Path, str]) path to the csv file. @@ -59,22 +57,22 @@ class RawCSVReader(IRawDataProvider): time_columns = _match_columns(columns, TIME_COLUMN_NAMES) - if timestamp_column_name is None: - timestamp_columns = _match_columns(columns, TIMESTAMP_KEY_COLUMN_NAMES) - else: - timestamp_columns = _match_columns(columns, [timestamp_column_name]) + timestamp_columns = _match_columns(columns, self._timestamp_column_names) assert len(timestamp_columns) == 1, f'Cannot specify timestamp column, found column names: {columns}' timestamp_column_name = timestamp_columns[0] # Read from file = be aware of high memory usage self.__arr = pd.read_csv( - path=path, + path, parse_dates=time_columns ) assert self.__arr[timestamp_column_name].is_monotonic_increasing, 'Timestamps in column are not increasing' + assert self.__arr[timestamp_column_name].is_unique, 'Found >=2 equal timestamps' - for key, name in self.__arr.apply(lambda x: x.name).to_dict().items(): + self.__column_types = {} + + for key, name in self.__arr.dtypes.apply(lambda x: x.name).to_dict().items(): if name == 'object': new_value = str elif name.startswith('datetime'): @@ -99,6 +97,13 @@ class RawCSVReader(IRawDataProvider): """Column names mapping to it's types""" return self.__column_types + @staticmethod + def _convert_to_pytype(value): + if isinstance(value, pd._libs.tslibs.timestamps.Timestamp): + return value.to_pydatetime() + else: + return value + def reader(self) -> Generator[Iterable[type], None, None]: """Returns iterator over rows. @@ -106,7 +111,9 @@ class RawCSVReader(IRawDataProvider): Iterable[type]: values in the next row (order is the same as in column_names). """ for index, row in self.__arr.iterrows(): - yield row.values.tolist() + values = row.values.tolist() + values = map(RawCSVReader._convert_to_pytype, values) + yield list(values) def reader_annotated(self) -> Generator[Dict[str, type], None, None]: """Returns dict iterator over rows. @@ -117,6 +124,5 @@ class RawCSVReader(IRawDataProvider): for index, row in self.__arr.iterrows(): mapped_values = dict(row.to_dict()) for key, val in mapped_values.items(): - if isinstance(val, pd._libs.tslibs.timestamps.Timestamp): - mapped_values[key] = val.to_pydatetime() + mapped_values[key] = RawCSVReader._convert_to_pytype(val) yield mapped_values diff --git a/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py b/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py index 43c1313..def04a1 100644 --- a/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py +++ b/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py @@ -1,5 +1,8 @@ import abc -from typing import Tuple, Dict, Generator, Iterable +from typing import Tuple, Dict, Generator, Iterable, Optional, List + +TIME_COLUMN_NAMES = ['time', 'date', 'timestamp', 'datetime'] +TIMESTAMP_KEY_COLUMN_NAMES = ['timestamp', 'datetime'] class IRawDataProvider(abc.ABC): @@ -8,6 +11,15 @@ class IRawDataProvider(abc.ABC): increasing, non repetitive timestamps. Time between timestamps *may* vary.""" + _timestamp_column_names: List[str] + + def __init__(self, + timestamp_column_name: Optional[str] = None): + if timestamp_column_name: + self._timestamp_column_names = [timestamp_column_name] + else: + self._timestamp_column_names = TIMESTAMP_KEY_COLUMN_NAMES + @property @abc.abstractmethod def column_names(self) -> Tuple[str]: diff --git a/FCRgendata/tests/data_reader/data_reader_template.py b/FCRgendata/tests/data_reader/data_reader_template.py new file mode 100644 index 0000000..6b9b501 --- /dev/null +++ b/FCRgendata/tests/data_reader/data_reader_template.py @@ -0,0 +1,124 @@ +import abc +import csv +import datetime as dt +import io +from typing import Callable, Optional + +import pytest + +from FCRGenData.rawDataReader import IRawDataProvider + + +class RawDataProviderTestTemplate(abc.ABC): + @abc.abstractmethod + @pytest.yield_fixture + def reader_factory(self) -> Callable[[str, Optional[str]], IRawDataProvider]: + pass + + def test_import_basic(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() + dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + # unknown timestamp column + with pytest.raises(AssertionError) as exc: + reader = reader_factory(s.read()) + + assert 'Cannot specify timestamp column' in exc.value.args[0] + + reader_factory(s.read(), timestamp_column_name='time') + + def test_time_desc(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() - dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + # unknown timestamp column + with pytest.raises(AssertionError) as exc: + reader = reader_factory(s.read(), timestamp_column_name='time') + + assert 'Timestamps in column are not increasing' in exc.value.args[0] + + def test_time_eq(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + t = dt.datetime.now() + for i in range(5): + row = [t, i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + # unknown timestamp column + with pytest.raises(AssertionError) as exc: + reader = reader_factory(s.read(), timestamp_column_name='time') + + assert 'Found >=2 equal timestamps' in exc.value.args[0] + + def test_column_getters(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() + dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + + reader: IRawDataProvider = reader_factory(s.read(), timestamp_column_name='time') + + assert len(reader.column_names) == len(header) + for reader_col, header_col in zip(reader.column_names, header): + assert reader_col == header_col + + col_types = reader.columns + + # check types + assert col_types['time'] == dt.datetime + for i in range(1, len(header)): + col = header[i] + assert col_types[col].__name__ == col + + def test_read(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() + dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + + reader: IRawDataProvider = reader_factory(s.read(), timestamp_column_name='time') + + row_gen = reader.reader() + for row, grow in zip(content, row_gen): + assert row == grow + + dict_gen = reader.reader_annotated() + for row, row_dict in zip(content, dict_gen): + for col_name, value in zip(header, row): + assert row_dict[col_name] == value diff --git a/FCRgendata/tests/data_reader/test_csv_reader.py b/FCRgendata/tests/data_reader/test_csv_reader.py new file mode 100644 index 0000000..6f6436a --- /dev/null +++ b/FCRgendata/tests/data_reader/test_csv_reader.py @@ -0,0 +1,28 @@ +import tempfile +from typing import Callable + +import pytest + +from FCRGenData.rawDataReader import RawCSVReader +from FCRgendata.tests.data_reader.data_reader_template import RawDataProviderTestTemplate + + +class TestCSVReader(RawDataProviderTestTemplate): + + @pytest.yield_fixture + def empty_reader(self) -> RawCSVReader: + tfile = tempfile.NamedTemporaryFile(suffix='.csv') + yield RawCSVReader(tfile.name) + tfile.close() + + @pytest.yield_fixture + def reader_factory(self) -> Callable[[str], RawCSVReader]: + tfile = tempfile.NamedTemporaryFile(suffix='.csv', mode='w') + path = tfile.name + + def _write(s, *args, **kwargs): + tfile.write(s) + tfile.flush() + return RawCSVReader(path, *args, **kwargs) + + yield _write diff --git a/FCRgendata/tests/test_data_reader.py b/FCRgendata/tests/test_data_reader.py deleted file mode 100644 index d7dd395..0000000 --- a/FCRgendata/tests/test_data_reader.py +++ /dev/null @@ -1,7 +0,0 @@ -import abc -import pytest - - -class RawDataProviderTestTemplate(abc.ABC): - pass - # TODO tests common for each implementation \ No newline at end of file -- GitLab