diff --git a/FCRdataLoader/Pipfile b/FCRdataLoader/Pipfile new file mode 100644 index 0000000000000000000000000000000000000000..43457cfbe44754865aa98933b7ebbd5706164d35 --- /dev/null +++ b/FCRdataLoader/Pipfile @@ -0,0 +1,14 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +pipenv-setup = "*" + +[packages] +numpy = "*" +torch = "*" + +[requires] +python_version = "3.8" diff --git a/FCRdataLoader/Pipfile.lock b/FCRdataLoader/Pipfile.lock new file mode 100644 index 0000000000000000000000000000000000000000..2b43faa085f3a34fbc38e067361b84398b272829 --- /dev/null +++ b/FCRdataLoader/Pipfile.lock @@ -0,0 +1,393 @@ +{ + "_meta": { + "hash": { + "sha256": "97a59620fc51f3a45a147fa7068de11884d82d7f6b6516eab9e7ee9b0ce22cbf" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "numpy": { + "hashes": [ + "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94", + "sha256:06fab248a088e439402141ea04f0fffb203723148f6ee791e9c75b3e9e82f080", + "sha256:0eef32ca3132a48e43f6a0f5a82cb508f22ce5a3d6f67a8329c81c8e226d3f6e", + "sha256:1ded4fce9cfaaf24e7a0ab51b7a87be9038ea1ace7f34b841fe3b6894c721d1c", + "sha256:2e55195bc1c6b705bfd8ad6f288b38b11b1af32f3c8289d6c50d47f950c12e76", + "sha256:2ea52bd92ab9f768cc64a4c3ef8f4b2580a17af0a5436f6126b08efbd1838371", + "sha256:36674959eed6957e61f11c912f71e78857a8d0604171dfd9ce9ad5cbf41c511c", + "sha256:384ec0463d1c2671170901994aeb6dce126de0a95ccc3976c43b0038a37329c2", + "sha256:39b70c19ec771805081578cc936bbe95336798b7edf4732ed102e7a43ec5c07a", + "sha256:400580cbd3cff6ffa6293df2278c75aef2d58d8d93d3c5614cd67981dae68ceb", + "sha256:43d4c81d5ffdff6bae58d66a3cd7f54a7acd9a0e7b18d97abb255defc09e3140", + "sha256:50a4a0ad0111cc1b71fa32dedd05fa239f7fb5a43a40663269bb5dc7877cfd28", + "sha256:603aa0706be710eea8884af807b1b3bc9fb2e49b9f4da439e76000f3b3c6ff0f", + "sha256:6149a185cece5ee78d1d196938b2a8f9d09f5a5ebfbba66969302a778d5ddd1d", + "sha256:759e4095edc3c1b3ac031f34d9459fa781777a93ccc633a472a5468587a190ff", + "sha256:7fb43004bce0ca31d8f13a6eb5e943fa73371381e53f7074ed21a4cb786c32f8", + "sha256:811daee36a58dc79cf3d8bdd4a490e4277d0e4b7d103a001a4e73ddb48e7e6aa", + "sha256:8b5e972b43c8fc27d56550b4120fe6257fdc15f9301914380b27f74856299fea", + "sha256:99abf4f353c3d1a0c7a5f27699482c987cf663b1eac20db59b8c7b061eabd7fc", + "sha256:a0d53e51a6cb6f0d9082decb7a4cb6dfb33055308c4c44f53103c073f649af73", + "sha256:a12ff4c8ddfee61f90a1633a4c4afd3f7bcb32b11c52026c92a12e1325922d0d", + "sha256:a4646724fba402aa7504cd48b4b50e783296b5e10a524c7a6da62e4a8ac9698d", + "sha256:a76f502430dd98d7546e1ea2250a7360c065a5fdea52b2dffe8ae7180909b6f4", + "sha256:a9d17f2be3b427fbb2bce61e596cf555d6f8a56c222bd2ca148baeeb5e5c783c", + "sha256:ab83f24d5c52d60dbc8cd0528759532736b56db58adaa7b5f1f76ad551416a1e", + "sha256:aeb9ed923be74e659984e321f609b9ba54a48354bfd168d21a2b072ed1e833ea", + "sha256:c843b3f50d1ab7361ca4f0b3639bf691569493a56808a0b0c54a051d260b7dbd", + "sha256:cae865b1cae1ec2663d8ea56ef6ff185bad091a5e33ebbadd98de2cfa3fa668f", + "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff", + "sha256:cf2402002d3d9f91c8b01e66fbb436a4ed01c6498fffed0e4c7566da1d40ee1e", + "sha256:d051ec1c64b85ecc69531e1137bb9751c6830772ee5c1c426dbcfe98ef5788d7", + "sha256:d6631f2e867676b13026e2846180e2c13c1e11289d67da08d71cacb2cd93d4aa", + "sha256:dbd18bcf4889b720ba13a27ec2f2aac1981bd41203b3a3b27ba7a33f88ae4827", + "sha256:df609c82f18c5b9f6cb97271f03315ff0dbe481a2a02e56aeb1b1a985ce38e60" + ], + "index": "pypi", + "version": "==1.19.5" + }, + "torch": { + "hashes": [ + "sha256:2e49cac969976be63117004ee00d0a3e3dd4ea662ad77383f671b8992825de1a", + "sha256:38d67f4fb189a92a977b2c0a38e4f6dd413e0bf55aa6d40004696df7e40a71ff", + "sha256:422e64e98d0e100c360993819d0307e5d56e9517b26135808ad68984d577d75a", + "sha256:5d76c255a41484c1d41a9ff570b9c9f36cb85df9428aa15a58ae16ac7cfc2ea6", + "sha256:6652a767a0572ae0feb74ad128758e507afd3b8396b6e7f147e438ba8d4c6f63", + "sha256:a3793dcceb12b1e2281290cca1277c5ce86ddfd5bf044f654285a4d69057aea7", + "sha256:af464a6f4314a875035e0c4c2b07517599704b214634f4ed3ad2e748c5ef291f", + "sha256:d241c3f1c4d563e4ba86f84769c23e12606db167ee6f674eedff6d02901462e3", + "sha256:dd2fc6880c95e836960d86efbbc7f63d3287f2e1893c51d31f96dbfe02f0d73e", + "sha256:de84b4166e3f7335eb868b51d3bbd909ec33828af27290b4171bce832a55be3c", + "sha256:e000b94be3aa58ad7f61e7d07cf379ea9366cf6c6874e68bd58ad0bdc537b3a7", + "sha256:f0aaf657145533824b15f2fd8fde8f8c67fe6c6281088ef588091f03fad90243" + ], + "index": "pypi", + "version": "==1.7.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + } + }, + "develop": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.3.0" + }, + "black": { + "hashes": [ + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" + ], + "markers": "python_version >= '3.6'", + "version": "==19.10b0" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "cerberus": { + "hashes": [ + "sha256:302e6694f206dd85cb63f13fd5025b31ab6d38c99c50c6d769f8fa0b0f299589" + ], + "version": "==1.3.2" + }, + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.4" + }, + "distlib": { + "hashes": [ + "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", + "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" + ], + "version": "==0.3.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "orderedmultidict": { + "hashes": [ + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" + ], + "version": "==1.0.1" + }, + "packaging": { + "hashes": [ + "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858", + "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.8" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, + "pep517": { + "hashes": [ + "sha256:3985b91ebf576883efe5fa501f42a16de2607684f3797ddba7202b71b7d0da51", + "sha256:aeb78601f2d1aa461960b43add204cc7955667687fbcf9cdb5170f00556f117f" + ], + "version": "==0.9.1" + }, + "pip-shims": { + "hashes": [ + "sha256:05b00ade9d1e686a98bb656dd9b0608a933897283dc21913fad6ea5409ff7e91", + "sha256:16ca9f87485667b16b978b68a1aae4f9cc082c0fa018aed28567f9f34a590569" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.5.3" + }, + "pipenv-setup": { + "hashes": [ + "sha256:8a439aff7b16e18d7e07702c9186fc5fe86156679eace90e10c2578a43bd7af1", + "sha256:e1bfd55c1152024e762f1c17f6189fcb073166509e7c0228870f7ea160355648" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pipfile": { + "hashes": [ + "sha256:f7d9f15de8b660986557eb3cc5391aa1a16207ac41bc378d03f414762d36c984" + ], + "version": "==0.0.2" + }, + "plette": { + "extras": [ + "validation" + ], + "hashes": [ + "sha256:46402c03e36d6eadddad2a5125990e322dd74f98160c8f2dcd832b2291858a26", + "sha256:d6c9b96981b347bddd333910b753b6091a2c1eb2ef85bb373b4a67c9d91dca16" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.3" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.1" + }, + "regex": { + "hashes": [ + "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", + "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", + "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", + "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", + "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", + "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", + "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", + "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", + "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", + "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", + "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", + "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", + "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", + "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", + "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", + "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", + "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", + "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", + "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", + "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", + "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", + "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", + "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", + "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", + "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", + "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", + "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", + "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", + "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", + "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", + "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", + "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", + "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", + "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", + "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", + "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", + "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", + "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", + "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", + "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", + "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + ], + "version": "==2020.11.13" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.25.1" + }, + "requirementslib": { + "hashes": [ + "sha256:50d20f27e4515a2393695b0d886219598302163438ae054253147b2bad9b4a44", + "sha256:9c1e8666ca4512724cdd1739adcc7df19ec7ad2ed21f0e748f9631ad6b54f321" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.5.16" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "tomlkit": { + "hashes": [ + "sha256:6babbd33b17d5c9691896b0e68159215a9387ebfa938aa3ac42f4a4beeb2b831", + "sha256:ac57f29693fab3e309ea789252fcce3061e19110085aa31af5446ca749325618" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.7.0" + }, + "typed-ast": { + "hashes": [ + "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1", + "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d", + "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6", + "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd", + "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37", + "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151", + "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07", + "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440", + "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70", + "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496", + "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea", + "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400", + "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc", + "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606", + "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc", + "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581", + "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412", + "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a", + "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2", + "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787", + "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f", + "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937", + "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64", + "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487", + "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b", + "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41", + "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a", + "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3", + "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166", + "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10" + ], + "version": "==1.4.2" + }, + "urllib3": { + "hashes": [ + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.2" + }, + "vistir": { + "hashes": [ + "sha256:a37079cdbd85d31a41cdd18457fe521e15ec08b255811e81aa061fd5f48a20fb", + "sha256:eff1d19ef50c703a329ed294e5ec0b0fbb35b96c1b3ee6dcdb266dddbe1e935a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.2" + }, + "wheel": { + "hashes": [ + "sha256:78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", + "sha256:e11eefd162658ea59a60a0f6c7d493a7190ea4b9a85e335b33489d9f17e0245e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.36.2" + } + } +} diff --git a/FCRdataLoader/README.md b/FCRdataLoader/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5e16c6c6a989259d03fe188b80e565199811d598 --- /dev/null +++ b/FCRdataLoader/README.md @@ -0,0 +1,53 @@ +# pytorch dataset for FCR app (currently uses only data from single file) +# How to install dependencies + - make sure you have python3 installed + - if you don't have pipenv already installed run: +```sh +$ pip install pipenv +``` + - now run in directory with `Pipfile` +```sh +$ pipenv install +$ pipenv lock -r > requirements.txt +$ pip install -r requirements.txt +``` + +

+ +# Contents +## FCRdataLoader.fcrdataloader.dataset.FCRtrainDataSet + - containing training data +## FCRdataLoader.fcrdataloader.dataset.FCRtestDataSet + - containing test data +### details about pytorch dataset and DataLoader can be found here: https://pytorch.org/docs/stable/data.html + +# Tests + - to run automatic tests run +```sh +$ python setup.py test +``` + +

+ +# Example of usage +```Python +from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet +from torch.utils.data import DataLoader + + +train_data = FCRtrainDataSet(50, 5) +loader = DataLoader(dataset=train_data, batch_size=16, shuffle=True) + +model = Network() # some network +criterion = nn.MSELoss() +optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) + +for epoch in range(100): + for (seq, answer) in enumerate(dataloader): + prediction = model(seq) + loss = criterion(prediction, answer) + + optimizer.zero_grad() + loss.backward() + optimizer.step() +``` \ No newline at end of file diff --git a/FCRdataLoader/data/data2.csv b/FCRdataLoader/data/data2.csv new file mode 100644 index 0000000000000000000000000000000000000000..aa2ee65f0921a1ea188027ed80bbd59de5e8c3b5 --- /dev/null +++ b/FCRdataLoader/data/data2.csv @@ -0,0 +1,218 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-11-27T12:44:35.519,6.0,9.435382843017578,0,1,0,1,0,1,0 +2020-11-27T12:45:35.519,17.0,8.48339557647705,0,1,0,1,0,1,0 +2020-11-27T12:46:35.519,10.0,23.755578994750977,0,1,0,1,0,1,0 +2020-11-27T12:47:35.519,5.0,11.223828315734863,0,1,0,1,0,1,0 +2020-11-27T12:48:35.519,43.0,7.788514137268066,0,1,0,1,0,1,0 +2020-11-27T12:49:35.519,10.0,44.337806701660156,0,1,0,1,0,1,0 +2020-11-27T12:50:35.519,4.0,8.92234992980957,0,1,0,1,0,1,0 +2020-11-27T12:51:35.519,5.0,7.031393051147461,0,1,0,1,0,1,0 +2020-11-27T12:52:35.519,6.0,7.168320655822754,0,1,0,1,0,1,0 +2020-11-27T12:53:35.519,8.0,7.513263702392578,0,1,0,1,0,1,0 +2020-11-27T12:54:35.519,4.0,9.204913139343262,0,1,0,1,0,1,0 +2020-11-27T12:55:35.519,14.0,6.85783052444458,0,1,0,1,0,1,0 +2020-11-27T12:56:35.519,17.33333333333333,20.1556453704834,0,1,0,1,0,1,0 +2020-11-27T12:57:35.519,12.0,15.963403701782227,0,1,0,1,0,1,0 +2020-11-27T12:58:35.519,4.0,12.1525297164917,0,1,0,1,0,1,0 +2020-11-27T12:59:35.519,4.0,6.540548324584961,0,1,0,1,0,1,0 +2020-11-27T13:00:35.519,6.0,5.795986652374268,0,1,0,1,0,1,0 +2020-11-27T13:01:35.519,7.0,6.7676005363464355,0,1,0,1,0,1,0 +2020-11-27T13:02:35.519,13.0,8.009368896484375,0,1,0,1,0,1,0 +2020-11-27T13:03:35.519,6.0,16.15487289428711,0,1,0,1,0,1,0 +2020-11-27T13:04:35.519,12.0,7.246679306030273,0,1,0,1,0,1,0 +2020-11-27T13:05:35.519,6.0,16.62491798400879,0,1,0,1,0,1,0 +2020-11-27T13:06:35.519,5.0,7.308323860168457,0,1,0,1,0,1,0 +2020-11-27T13:07:35.519,31.5,6.282835483551025,0,1,0,1,0,1,0 +2020-11-27T13:08:35.519,4.0,19.173612594604492,0,1,0,1,0,1,0 +2020-11-27T13:09:35.519,8.0,6.749720573425293,0,1,0,1,0,1,0 +2020-11-27T13:10:35.519,15.0,9.082990646362305,0,1,0,1,0,1,0 +2020-11-27T13:11:35.519,6.0,18.638320922851562,0,1,0,1,0,1,0 +2020-11-27T13:12:35.519,5.0,8.155741691589355,0,1,0,1,0,1,0 +2020-11-27T13:13:35.520,11.0,7.225281715393066,0,1,0,1,0,1,0 +2020-11-27T13:14:35.519,15.0,13.923819541931152,0,1,0,1,0,1,0 +2020-11-27T13:15:35.519,5.0,17.63955307006836,0,1,0,1,0,1,0 +2020-11-27T13:16:35.519,4.0,7.650225639343262,0,1,0,1,0,1,0 +2020-11-27T13:17:35.519,0.0,6.992892742156982,0,1,0,1,0,1,0 +2020-11-27T13:18:35.519,4.0,4.934108257293701,0,1,0,1,0,1,0 +2020-11-27T13:19:35.519,7.0,4.724740505218506,0,1,0,1,0,1,0 +2020-11-27T13:20:35.519,15.0,7.635990142822266,0,1,0,1,0,1,0 +2020-11-27T13:21:35.519,160.0,9.775514602661133,0,3,0,1,0,1,0 +2020-11-27T13:22:35.519,50.0,63.986534118652344,0,1,0,1,0,1,0 +2020-11-27T13:23:35.519,7.0,11.747467994689941,0,1,0,1,0,1,0 +2020-11-27T13:24:35.519,6.0,9.03959846496582,0,1,0,1,0,1,0 +2020-11-27T13:25:35.519,96.0,7.851625442504883,0,2,0,1,0,1,0 +2020-11-27T13:26:35.519,4.0,39.70072555541992,0,1,0,1,0,1,0 +2020-11-27T13:27:35.519,4.0,9.647521018981934,0,1,0,1,0,1,0 +2020-11-27T13:28:35.519,5.0,9.178874015808105,0,1,0,1,0,1,0 +2020-11-27T13:29:35.519,6.0,10.066906929016113,0,1,0,1,0,1,0 +2020-11-27T13:30:35.519,81.0,10.560870170593262,0,2,0,1,0,1,0 +2020-11-27T13:31:35.519,988.0,104.04701232910156,0,20,0,1,0,1,0 +2020-11-27T13:32:35.519,6.0,380.4679870605469,0,1,0,1,0,1,0 +2020-11-27T13:33:35.519,6.0,14.513042449951172,0,1,0,1,0,1,0 +2020-11-27T13:34:35.519,5.0,10.953410148620605,0,1,0,1,0,1,0 +2020-11-27T13:35:35.519,14.0,9.55089282989502,0,1,0,1,0,1,0 +2020-11-27T13:36:35.519,5.0,12.438122749328613,0,1,0,1,0,1,0 +2020-11-27T13:37:35.519,6.0,7.341762542724609,0,1,0,1,0,1,0 +2020-11-27T13:38:35.519,6.0,6.660945892333984,0,1,0,1,0,1,0 +2020-11-27T13:39:35.519,6.0,6.816884517669678,0,1,0,1,0,1,0 +2020-11-27T13:40:35.519,7.0,6.6493377685546875,0,1,0,1,0,1,0 +2020-11-27T13:41:35.519,8.0,5.92299222946167,0,1,0,1,0,1,0 +2020-11-27T13:42:35.519,6.0,5.889991283416748,0,1,0,1,0,1,0 +2020-11-27T13:43:35.519,5.0,6.110908508300781,0,1,0,1,0,1,0 +2020-11-27T13:44:35.519,4.0,5.786072254180908,0,1,0,1,0,1,0 +2020-11-27T13:45:35.519,0.0,5.522675514221191,0,1,0,1,0,1,0 +2020-11-27T13:46:35.519,8.0,4.02607536315918,0,1,0,1,0,1,0 +2020-11-27T13:47:35.519,6.0,11.456418991088867,0,1,0,1,0,1,0 +2020-11-27T13:48:35.519,0.0,5.000259876251221,0,1,0,1,0,1,0 +2020-11-27T13:49:35.519,43.0,3.127312660217285,0,1,0,1,0,1,0 +2020-11-27T13:50:35.519,0.0,30.338163375854492,0,1,0,1,0,1,0 +2020-11-27T13:51:35.521,5.0,2.966379165649414,0,1,0,1,0,1,0 +2020-11-27T13:52:35.519,5.0,3.754279613494873,0,1,0,1,0,1,0 +2020-11-27T13:53:35.519,0.0,3.5321502685546875,0,1,0,1,0,1,0 +2020-11-27T13:54:35.519,6.0,2.0308921337127686,0,1,0,1,0,1,0 +2020-11-27T13:55:35.519,6.0,4.462838172912598,0,1,0,1,0,1,0 +2020-11-27T13:56:35.519,0.0,4.041638374328613,0,1,0,1,0,1,0 +2020-11-27T13:57:35.519,7.0,2.098271369934082,0,1,0,1,0,1,0 +2020-11-27T13:58:35.519,105.5,5.880876064300537,0,2,0,1,0,1,0 +2020-11-27T13:59:35.519,0.0,110.12590026855469,0,1,0,1,0,1,0 +2020-11-27T14:00:35.519,287.0,2.413583278656006,0,6,0,1,0,1,0 +2020-11-27T14:01:35.519,14.0,335.0638732910156,0,1,0,1,0,1,0 +2020-11-27T14:02:35.519,5.0,5.429197311401367,0,1,0,1,0,1,0 +2020-11-27T14:03:35.519,8.0,4.946113109588623,0,1,0,1,0,1,0 +2020-11-27T14:04:35.519,5.0,6.548556327819824,0,1,0,1,0,1,0 +2020-11-27T14:05:35.519,6.0,5.965693950653076,0,1,0,1,0,1,0 +2020-11-27T14:06:35.519,726.0,6.134999752044678,0,14,0,1,0,1,0 +2020-11-27T14:07:35.519,4665.0,1021.2295532226562,0,47,0,1,0,1,0 +2020-11-27T14:08:35.519,0.0,3070.738037109375,0,1,0,1,0,1,0 +2020-11-27T14:09:35.520,9.0,14.447136878967285,0,1,0,1,0,1,0 +2020-11-27T14:10:35.519,5.0,10.972953796386719,0,1,0,1,0,1,0 +2020-11-27T14:11:35.519,4.0,7.169851303100586,0,1,0,1,0,1,0 +2020-11-27T14:12:35.519,7.0,5.326818943023682,0,1,0,1,0,1,0 +2020-11-27T14:13:35.519,6.0,5.767587661743164,0,1,0,1,0,1,0 +2020-11-27T14:14:35.519,6.0,5.402779579162598,0,1,0,1,0,1,0 +2020-11-27T14:15:35.519,6.0,5.32569694519043,0,1,0,1,0,1,0 +2020-11-27T14:16:35.519,18.0,4.896035671234131,0,1,0,1,0,1,0 +2020-11-27T14:17:35.519,3209.0,6.206113815307617,0,47,0,1,0,1,0 +2020-11-27T14:18:35.519,5.0,591.1841430664062,0,1,0,1,0,1,0 +2020-11-27T14:19:35.519,4.0,5.264420986175537,0,1,0,1,0,1,0 +2020-11-27T14:20:35.519,4.0,3.9137449264526367,0,1,0,1,0,1,0 +2020-11-27T14:21:35.519,6.0,3.76383638381958,0,1,0,1,0,1,0 +2020-11-27T14:22:35.519,6.0,4.28965950012207,0,1,0,1,0,1,0 +2020-11-27T14:23:35.519,5.0,4.339838027954102,0,1,0,1,0,1,0 +2020-11-27T14:24:35.519,15.5,4.126605033874512,0,1,0,1,0,1,0 +2020-11-27T14:25:35.519,16.0,6.732624530792236,0,1,0,1,0,1,0 +2020-11-27T14:26:35.519,7.0,7.165763854980469,0,1,0,1,0,1,0 +2020-11-27T14:27:35.519,5.0,5.099540710449219,0,1,0,1,0,1,0 +2020-11-27T14:28:35.519,4.0,4.222059726715088,0,1,0,1,0,1,0 +2020-11-27T14:29:35.519,4.0,4.934439659118652,0,1,0,1,0,1,0 +2020-11-27T14:30:35.519,5.0,4.902071475982666,0,1,0,1,0,1,0 +2020-11-27T14:31:35.519,7.0,5.144791603088379,0,1,0,1,0,1,0 +2020-11-27T14:32:35.519,26.0,6.1177077293396,0,1,0,1,0,1,0 +2020-11-27T14:33:35.519,2638.5,28.131595611572266,0,47,0,1,0,1,0 +2020-11-27T14:34:35.519,1067.0,616.666748046875,0,21,0,1,0,1,0 +2020-11-27T14:35:35.519,21.0,101.96915435791016,0,1,0,1,0,1,0 +2020-11-27T14:36:35.519,5.0,8.676255226135254,0,1,0,1,0,1,0 +2020-11-27T14:37:35.519,115.0,6.9384965896606445,0,2,0,1,0,1,0 +2020-11-27T14:38:35.519,5.0,57.02287292480469,0,1,0,1,0,1,0 +2020-11-27T14:39:35.519,7.0,11.969968795776367,0,1,0,1,0,1,0 +2020-11-27T14:40:35.519,6.0,12.453564643859863,0,1,0,1,0,1,0 +2020-11-27T14:41:35.519,7.0,12.684252738952637,0,1,0,1,0,1,0 +2020-11-27T14:42:35.519,7.0,12.762392044067383,0,1,0,1,0,1,0 +2020-11-27T14:43:35.519,6.0,10.544450759887695,0,1,0,1,0,1,0 +2020-11-27T14:44:35.519,16.0,7.521728515625,0,1,0,1,0,1,0 +2020-11-27T14:45:35.519,5.0,9.791584014892578,0,1,0,1,0,1,0 +2020-11-27T14:46:35.519,8.0,6.312222480773926,0,1,0,1,0,1,0 +2020-11-27T14:47:35.519,4.0,6.674623966217041,0,1,0,1,0,1,0 +2020-11-27T14:48:35.519,5.0,5.500234127044678,0,1,0,1,0,1,0 +2020-11-27T14:49:35.519,7.0,5.7142205238342285,0,1,0,1,0,1,0 +2020-11-27T14:50:35.519,5.0,6.574368953704834,0,1,0,1,0,1,0 +2020-11-27T14:51:35.519,4.0,5.641943454742432,0,1,0,1,0,1,0 +2020-11-27T14:52:35.519,48.0,5.242447853088379,0,1,0,1,0,1,0 +2020-11-27T14:53:35.519,10.0,31.18132781982422,0,1,0,1,0,1,0 +2020-11-27T14:54:35.519,17.0,6.470431327819824,0,1,0,1,0,1,0 +2020-11-27T14:55:35.519,92.0,17.268766403198242,0,2,0,1,0,1,0 +2020-11-27T14:56:35.519,5.0,94.94540405273438,0,1,0,1,0,1,0 +2020-11-27T14:57:35.519,0.0,6.798692226409912,0,1,0,1,0,1,0 +2020-11-27T14:58:35.519,5.0,5.002079963684082,0,1,0,1,0,1,0 +2020-11-27T14:59:35.519,5.0,5.192873477935791,0,1,0,1,0,1,0 +2020-11-27T15:00:35.519,47.0,5.093393325805664,0,1,0,1,0,1,0 +2020-11-27T15:01:35.519,6.0,66.24467468261719,0,1,0,1,0,1,0 +2020-11-27T15:02:35.519,106.5,6.989488124847412,0,2,0,1,0,1,0 +2020-11-27T15:03:35.519,18.0,114.1700668334961,0,1,0,1,0,1,0 +2020-11-27T15:04:35.519,4.0,9.336390495300293,0,1,0,1,0,1,0 +2020-11-27T15:05:35.519,5.0,6.51580810546875,0,1,0,1,0,1,0 +2020-11-27T15:06:35.519,8.0,5.369061470031738,0,1,0,1,0,1,0 +2020-11-27T15:07:35.519,7.0,5.92508602142334,0,1,0,1,0,1,0 +2020-11-27T15:08:35.519,6.0,6.801459312438965,0,1,0,1,0,1,0 +2020-11-27T15:09:35.519,5.0,7.037836074829102,0,1,0,1,0,1,0 +2020-11-27T15:10:35.519,6.5,6.7276225090026855,0,1,0,1,0,1,0 +2020-11-27T15:11:35.519,6.0,6.155050754547119,0,1,0,1,0,1,0 +2020-11-27T15:12:35.519,65.0,5.615631103515625,0,1,0,1,0,1,0 +2020-11-27T15:13:35.519,5.0,59.99169921875,0,1,0,1,0,1,0 +2020-11-27T15:14:35.519,0.0,5.508193492889404,0,1,0,1,0,1,0 +2020-11-27T15:15:35.519,7.0,3.9370083808898926,1,1,0,1,0,1,0 +2020-11-27T15:16:35.519,6.0,5.107851505279541,1,1,0,1,0,1,0 +2020-11-27T15:17:35.519,0.0,4.678167343139648,1,1,0,1,0,1,0 +2020-11-27T15:18:35.519,6.0,2.993084669113159,1,1,0,1,0,1,0 +2020-11-27T15:19:35.519,0.0,4.081082344055176,1,1,0,1,0,1,0 +2020-11-27T15:20:35.519,0.0,2.3085193634033203,1,1,0,1,0,1,0 +2020-11-27T15:21:35.519,0.0,1.4350025653839111,1,1,0,1,0,1,0 +2020-11-27T15:22:35.519,0.0,0.8783214092254639,1,1,0,1,0,1,0 +2020-11-27T15:23:35.519,0.0,0.5170314311981201,1,1,0,1,0,1,0 +2020-11-27T15:24:35.519,0.0,0.30115222930908203,1,1,0,1,0,1,0 +2020-11-27T15:25:35.519,0.0,0.19430434703826904,1,1,0,1,0,1,0 +2020-11-27T15:26:35.519,0.0,0.06121242046356201,1,1,0,1,0,1,0 +2020-11-27T15:27:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:28:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:29:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:30:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:31:35.520,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:32:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:33:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:34:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:35:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:36:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:37:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:38:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:39:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:40:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:41:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:42:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:43:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:44:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:45:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:46:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:47:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:48:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:49:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:50:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:51:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:52:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:53:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:54:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:55:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:56:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:57:35.519,0.0,0.0,1,1,0,1,0,1,0 +2020-11-27T15:58:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T15:59:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:00:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:01:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:02:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:03:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:04:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:05:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:06:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:07:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:08:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:09:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:10:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:11:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:12:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:13:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:14:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:15:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:16:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:17:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:18:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:19:35.519,0.0,0.0,-1,1,0,1,0,1,0 +2020-11-27T16:20:35.519,0.0,0.0,-1,1,0,1,0,1,0 diff --git a/FCRdataLoader/data/data4.csv b/FCRdataLoader/data/data4.csv new file mode 100644 index 0000000000000000000000000000000000000000..0d25180e3b29fc28276c0a86d11402d6b37df29f --- /dev/null +++ b/FCRdataLoader/data/data4.csv @@ -0,0 +1,116 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-12-15T11:49:00,0.0,2.880878210067749,0,1,0,1,0,1,0 +2020-12-15T11:50:00,0.0,1.698030710220337,0,1,0,1,0,1,0 +2020-12-15T11:51:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T11:52:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T11:53:00,0.0,0.3707929849624634,0,1,0,1,0,1,0 +2020-12-15T11:54:00,0.0,0.45026612281799316,0,1,0,1,0,1,0 +2020-12-15T11:55:00,0.0,0.41281116008758545,0,1,0,1,0,1,0 +2020-12-15T11:56:00,0.0,0.3778296709060669,0,1,0,1,0,1,0 +2020-12-15T11:57:00,0.0,0.42775797843933105,0,1,0,1,0,1,0 +2020-12-15T11:58:00,0.0,52.15424346923828,0,1,0,1,0,1,0 +2020-12-15T11:59:00,0.0,1.3606882095336914,0,1,0,1,0,1,0 +2020-12-15T12:00:00,0.0,1.7871627807617188,0,1,0,1,0,1,0 +2020-12-15T12:01:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:02:00,0.0,0.29342174530029297,0,1,0,1,0,1,0 +2020-12-15T12:03:00,0.0,0.22346079349517822,0,1,0,1,0,1,0 +2020-12-15T12:04:00,0.0,0.20592844486236572,0,1,0,1,0,1,0 +2020-12-15T12:05:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:06:00,0.0,0.2854079008102417,0,1,0,1,0,1,0 +2020-12-15T12:07:00,0.0,2.0263261795043945,0,1,0,1,0,1,0 +2020-12-15T12:08:00,0.0,1.193394660949707,0,1,0,1,0,1,0 +2020-12-15T12:09:00,0.0,0.6382132768630981,0,1,0,1,0,1,0 +2020-12-15T12:10:00,0.0,14.562797546386719,0,1,0,1,0,1,0 +2020-12-15T12:11:00,0.0,0.7512111663818359,0,1,0,1,0,1,0 +2020-12-15T12:12:00,0.0,0.8914639949798584,0,1,0,1,0,1,0 +2020-12-15T12:13:00,0.0,0.7688208818435669,0,1,0,1,0,1,0 +2020-12-15T12:14:00,0.0,0.7653945684432983,0,1,0,1,0,1,0 +2020-12-15T12:15:00,0.0,1.1794123649597168,0,1,0,1,0,1,0 +2020-12-15T12:16:00,0.0,1.1221184730529785,0,1,0,1,0,1,0 +2020-12-15T12:17:00,0.0,1.2044415473937988,0,1,0,1,0,1,0 +2020-12-15T12:18:00,0.0,1.167661190032959,0,1,0,1,0,1,0 +2020-12-15T12:19:00,0.0,29.40463638305664,0,1,0,1,0,1,0 +2020-12-15T12:20:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:21:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:22:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:23:00,0.0,0.26250433921813965,0,1,0,1,0,1,0 +2020-12-15T12:24:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:25:00,0.0,0.01861274242401123,0,1,0,1,0,1,0 +2020-12-15T12:26:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:27:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:28:00,0.0,1.031090259552002,0,1,0,1,0,1,0 +2020-12-15T12:29:00,0.0,0.1976233720779419,0,1,0,1,0,1,0 +2020-12-15T12:30:00,0.0,0.16222071647644043,0,1,0,1,0,1,0 +2020-12-15T12:31:00,0.0,0.45725953578948975,0,1,0,1,0,1,0 +2020-12-15T12:32:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:33:00,0.0,0.6110225915908813,0,1,0,1,0,1,0 +2020-12-15T12:34:00,0.0,0.5869172811508179,0,1,0,1,0,1,0 +2020-12-15T12:35:00,0.0,824.6790161132812,0,1,0,1,0,1,0 +2020-12-15T12:36:00,0.0,143.26446533203125,0,1,0,1,0,1,0 +2020-12-15T12:37:00,0.0,0.9533004760742188,0,1,0,1,0,1,0 +2020-12-15T12:38:00,0.0,6.044524192810059,0,1,0,1,0,1,0 +2020-12-15T12:39:00,0.0,2.429046630859375,0,1,0,1,0,1,0 +2020-12-15T12:40:00,0.0,2.7120466232299805,0,1,0,1,0,1,0 +2020-12-15T12:41:00,0.0,2.1224734783172607,0,1,0,1,0,1,0 +2020-12-15T12:42:00,0.0,1.8528714179992676,0,1,0,1,0,1,0 +2020-12-15T12:43:00,0.0,1.8902008533477783,0,1,0,1,0,1,0 +2020-12-15T12:44:00,0.0,1.9152851104736328,0,1,0,1,0,1,0 +2020-12-15T12:45:00,0.0,1.8422601222991943,0,1,0,1,0,1,0 +2020-12-15T12:46:00,0.0,32.213531494140625,0,1,0,1,0,1,0 +2020-12-15T12:47:00,0.0,4.091504096984863,0,1,0,1,0,1,0 +2020-12-15T12:48:00,0.0,10.508217811584473,0,1,0,1,0,1,0 +2020-12-15T12:49:00,0.0,2.708798885345459,0,1,0,1,0,1,0 +2020-12-15T12:50:00,0.0,2.3304004669189453,0,1,0,1,0,1,0 +2020-12-15T12:51:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T12:52:00,0.0,1.0770142078399658,0,1,0,1,0,1,0 +2020-12-15T12:53:00,0.0,0.8451101779937744,0,1,0,1,0,1,0 +2020-12-15T12:54:00,0.0,1.1430389881134033,0,1,0,1,0,1,0 +2020-12-15T12:55:00,0.0,1.2068374156951904,0,1,0,1,0,1,0 +2020-12-15T12:56:00,0.0,728.1805419921875,0,1,0,1,0,1,0 +2020-12-15T12:57:00,0.0,15.266721725463867,0,1,0,1,0,1,0 +2020-12-15T12:58:00,0.0,2.8867130279541016,0,1,0,1,0,1,0 +2020-12-15T12:59:00,0.0,2.095475196838379,0,1,0,1,0,1,0 +2020-12-15T13:00:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:01:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:02:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:03:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:04:00,0.0,0.0,0,1,0,1,0,1,0 +2020-12-15T13:05:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:06:00,0.0,204.33526611328125,1,1,0,1,0,1,0 +2020-12-15T13:07:00,0.0,2.2900924682617188,1,1,0,1,0,1,0 +2020-12-15T13:08:00,0.0,4.229484558105469,1,1,0,1,0,1,0 +2020-12-15T13:09:00,0.0,2.979954957962036,1,1,0,1,0,1,0 +2020-12-15T13:10:00,0.0,0.8080476522445679,1,1,0,1,0,1,0 +2020-12-15T13:11:00,0.0,2.4772040843963623,1,1,0,1,0,1,0 +2020-12-15T13:12:00,0.0,2.2220327854156494,1,1,0,1,0,1,0 +2020-12-15T13:13:00,0.0,365.6095275878906,1,1,0,1,0,1,0 +2020-12-15T13:14:00,0.0,2.6103553771972656,1,1,0,1,0,1,0 +2020-12-15T13:15:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:16:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:17:00,0.0,730.0831909179688,1,1,0,1,0,1,0 +2020-12-15T13:18:00,0.0,5.1552734375,1,1,0,1,0,1,0 +2020-12-15T13:19:00,0.0,5.315293312072754,1,1,0,1,0,1,0 +2020-12-15T13:20:00,0.0,3.747960090637207,1,1,0,1,0,1,0 +2020-12-15T13:21:00,0.0,3.213507652282715,1,1,0,1,0,1,0 +2020-12-15T13:22:00,0.0,3.209949493408203,1,1,0,1,0,1,0 +2020-12-15T13:23:00,0.0,12.009992599487305,1,1,0,1,0,1,0 +2020-12-15T13:24:00,0.0,68.0241928100586,1,1,0,1,0,1,0 +2020-12-15T13:25:00,0.0,5.560791492462158,1,1,0,1,0,1,0 +2020-12-15T13:26:00,0.0,4.588584899902344,1,1,0,1,0,1,0 +2020-12-15T13:27:00,0.0,2.873000144958496,1,1,0,1,0,1,0 +2020-12-15T13:28:00,0.0,2.874795436859131,1,1,0,1,0,1,0 +2020-12-15T13:29:00,0.0,2.6174628734588623,1,1,0,1,0,1,0 +2020-12-15T13:30:00,0.0,0.0,1,1,0,1,0,1,0 +2020-12-15T13:31:00,0.0,0.7121289968490601,-1,1,0,1,0,1,0 +2020-12-15T13:32:00,0.0,0.5510646104812622,-1,1,0,1,0,1,0 +2020-12-15T13:33:00,0.0,0.3965524435043335,-1,1,0,1,0,1,0 +2020-12-15T13:34:00,0.0,0.26016998291015625,-1,1,0,1,0,1,0 +2020-12-15T13:35:00,0.0,0.129294753074646,-1,1,0,1,0,1,0 +2020-12-15T13:36:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:37:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:38:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:39:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:40:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:41:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:42:00,0.0,0.0,-1,1,0,1,0,1,0 +2020-12-15T13:43:00,0.0,0.0,-1,1,0,1,0,1,0 diff --git a/FCRdataLoader/setup.py b/FCRdataLoader/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..5c3d74d8a0d754718ef1a6cbcd61600e4be2b43a --- /dev/null +++ b/FCRdataLoader/setup.py @@ -0,0 +1,157 @@ +"""A setuptools based setup module. +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +Modified by Madoshakalaka@Github (dependency links added) +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +from os import path + +# io.open is needed for projects that support Python 2.7 +# It ensures open() defaults to text mode with universal newlines, +# and accepts an argument to specify the text encoding +# Python 3 only projects can skip this import +from io import open + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, "README.md"), encoding="utf-8") as f: + long_description = f.read() + +# Arguments marked as "Required" below must be included for upload to PyPI. +# Fields marked as "Optional" may be commented out. + +setup( + # This is the name of your project. The first time you publish this + # package, this name will be registered for you. It will determine how + # users can install this project, e.g.: + # + # $ pip install sampleproject + # + # And where it will live on PyPI: https://pypi.org/project/sampleproject/ + # + # There are some restrictions on what makes a valid project name + # specification here: + # https://packaging.python.org/specifications/core-metadata/#name + name="FCRdataLoader", # Required + # Versions should comply with PEP 440: + # https://www.python.org/dev/peps/pep-0440/ + # + # For a discussion on single-sourcing the version across setup.py and the + # project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version="0.0.0", # Required + # This is a one-line description or tagline of what your project does. This + # corresponds to the "Summary" metadata field: + # https://packaging.python.org/specifications/core-metadata/#summary + # This is an optional longer description of your project that represents + # the body of text which users will see when they visit PyPI. + # + # Often, this is the same as your README, so you can just read it in from + # that file directly (as we have already done above) + # + # This field corresponds to the "Description" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-optional + # Denotes that our long_description is in Markdown; valid values are + # text/plain, text/x-rst, and text/markdown + # + # Optional if long_description is written in reStructuredText (rst) but + # required for plain-text or Markdown; if unspecified, "applications should + # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and + # fall back to text/plain if it is not valid rst" (see link below) + # + # This field corresponds to the "Description-Content-Type" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional + # This should be a valid link to your project's main homepage. + # + # This field corresponds to the "Home-Page" metadata field: + # https://packaging.python.org/specifications/core-metadata/#home-page-optional + # This should be your name or the name of the organization which owns the + # project. + # This should be a valid email address corresponding to the author listed + # above. + # Classifiers help users find your project by categorizing it. + # + # For a list of valid classifiers, see https://pypi.org/classifiers/ + + # This field adds keywords for your project which will appear on the + # project page. What does your project relate to? + # + # Note that this is a string of words separated by whitespace, not a list. + # You can just specify package directories manually here if your project is + # simple. Or you can use find_packages(). + # + # Alternatively, if you just want to distribute a single Python file, use + # the `py_modules` argument instead as follows, which will expect a file + # called `my_module.py` to exist: + # + # py_modules=["my_module"], + # + packages=find_packages(exclude=["contrib", "docs", "tests"]), # Required + # Specify which Python versions you support. In contrast to the + # 'Programming Language' classifiers above, 'pip install' will check this + # and refuse to install the project if the version does not match. If you + # do not support Python 2, you can simplify this to '>=3.5' or similar, see + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4", + # This field lists other packages that your project depends on to run. + # Any package you put here will be installed by pip when your project is + # installed, so they must be valid existing projects. + # + # For an analysis of "install_requires" vs pip's requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + "numpy==1.19.5", + "torch==1.7.1", + "typing-extensions==3.7.4.3", + ], # Optional + # List additional groups of dependencies here (e.g. development + # dependencies). Users will be able to install these using the "extras" + # syntax, for example: + # + # $ pip install sampleproject[dev] + # + # Similar to `install_requires` above, these must be valid existing + # projects. + # If there are data files included in your packages that need to be + # installed, specify them here. + # + # Sometimes you’ll want to use packages that are properly arranged with + # setuptools, but are not published to PyPI. In those cases, you can specify + # a list of one or more dependency_links URLs where the package can + # be downloaded, along with some additional hints, and setuptools + # will find and install the package correctly. + # see https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi + # + dependency_links=[], + # If using Python 2.6 or earlier, then these have to be included in + # MANIFEST.in as well. + # package_data={"sample": ["package_data.dat"]}, # Optional + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. See: + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files + # + # In this case, 'data_file' will be installed into '/my_data' + # data_files=[("my_data", ["data/data_file"])], # Optional + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # `pip` to create the appropriate form of executable for the target + # platform. + # + # For example, the following would provide a command called `sample` which + # executes the function `main` from this package when invoked: + # entry_points={"console_scripts": ["sample=sample:main"]}, # Optional + # List additional URLs that are relevant to your project as a dict. + # + # This field corresponds to the "Project-URL" metadata fields: + # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use + # + # Examples listed include a pattern for specifying where the package tracks + # issues, where the source is hosted, where to say thanks to the package + # maintainers, and where to support the project financially. The key is + # what's used to render the link text on PyPI. + test_suite="tests", +) diff --git a/FCRdataLoader/src/fcrdataloader/__init__.py b/FCRdataLoader/src/fcrdataloader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/FCRdataLoader/src/fcrdataloader/data.py b/FCRdataLoader/src/fcrdataloader/data.py new file mode 100644 index 0000000000000000000000000000000000000000..0a6d19419b4ae46af0d66931516309e1cee358e3 --- /dev/null +++ b/FCRdataLoader/src/fcrdataloader/data.py @@ -0,0 +1,5 @@ +from pathlib import Path + + +__DATA_DIR = Path(__file__).absolute().parents[1].joinpath('data') +DATA_FILES = sorted(__DATA_DIR.glob("**/*.csv")) diff --git a/FCRdataLoader/src/fcrdataloader/dataset.py b/FCRdataLoader/src/fcrdataloader/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..c27997bb7ab2212a411370c5531a7ef7ab37c47e --- /dev/null +++ b/FCRdataLoader/src/fcrdataloader/dataset.py @@ -0,0 +1,264 @@ +from torch.utils.data import Dataset, WeightedRandomSampler, DataLoader +from pathlib import Path +from typing import List, Callable +from re import search +import numpy as np +import torch + + +MAX_TEST_S = 1000 +TEST_TRAIN_RATIO = 0.3 + + +class SequenceForecastMultiDistributionDataset(Dataset): + """ + Pytorch subclass of dataset, represents dataset of + mapping sequence of inputs to single output with + some horizon. Also dataset is structured as set of + smaller datasets. It makes difference, with sequence + mapping becouse "boundaries" of datasets can not be + considered as valid sequence. + """ + + def __init__( + self, + start: int, + size: int, + getitem: Callable[[int], torch.Tensor], + ): + self.start = start + self.size = size + self.getitem = getitem + + def __len__(self): + return self.size + + def __getitem__(self, idx): + return self.getitem(self.start + idx) + + +class SequenceForecastMultiDistributionDatasetFactory: + """ + TODO update docs + TODO add input validation + TODO add tests??? + abstract class representing FCR dataset + Currently only uses data from one file + + must return tuple (x, y) where: + LEGEND: + t - timestamp + v(t) - avg rsp time at time t + p(t) - avg rsp time prediction at time t + split(t) - split value corresponding to prediction at time t + solv(t) - solution to cp-problem at time t (obtained from v(t)) + TUPLE: + x is np.array od dtype: float32 with values (v(t), p(t), split(t)), shape (n, 3) + y is np.array od dtype: float32 with values (solv(t)), shape (n, 6) + """ + + def __test_size(self, rows: int) -> int: + """ + given total rows of data returns test size + """ + + return min(MAX_TEST_S, int(TEST_TRAIN_RATIO * rows)) + + def __train_size(self, rows: int) -> int: + """ + given total rows of data returns train size + """ + return rows - self.__test_size(rows) + + def __init__( + self, + seq_len: int, + pred_step: int, + files: List[Path], + x_y_split: int, + usecols: List[int], + x_predictions_cols: List[int], + transforms=None, + delimiter=',', + skip_header=1 + ): + """ + seq_len: is number of following records retuned as a sequence + pred_step: is distance between last record in sequence and correctly predicted row + transforms: transformation applited input data. + if returned data is (x, y) then with transormation its (transorms(x), y) + usecols should contain ids of colums to be read from csv + in cols [0, x_y_split-1] we have intput cols mappings + in cols [x_y_split, end-1] we have output cols mapping + x_predictions_cols is list of cols from input that contain some prediction value + and they will be pushed in horizon creating preprocessing + + """ + + if seq_len < 0: + raise ValueError(f"seq_len can't be negative") + + if pred_step < 0: + raise ValueError(f"pred_step can't be negative") + + self.x = [None] * len(files) + self.y = [None] * len(files) + self.files = self.__sort_by_postfixnumb(files) + self.transforms = transforms + self.seq_len = seq_len + self.x_y_split = x_y_split + self.usecols = usecols + self.x_predictions_cols = x_predictions_cols + self.delimiter = delimiter + self.skip_header = skip_header + self.size = None + + self.__preproces_data(seq_len, pred_step) + + def load_series(self, file: str) -> (np.array, np.array): + series = np.genfromtxt( + file, + delimiter=self.delimiter, + skip_header=self.skip_header, + usecols=self.usecols, + dtype=np.float32 + ) + return np.hsplit(series, [self.x_y_split]) + + def __sort_by_postfixnumb(self, fnames: List[str]): + p = r'(\d+).' + return sorted( + fnames, + key=lambda x: int(search(p, x.name).groups()[-1]) + ) + + def __preproces_data(self, seq_len, pred_step): + sizes = np.zeros(len(self.files), dtype=np.int) + for i, f_path in enumerate(self.files): + x, y = self.load_series(f_path) # for now only one file + x_cols = np.hsplit(x, x.shape[1]) + + x_cols = self.__push_cols(x_cols, self.x_predictions_cols, pred_step) + x = np.hstack(x_cols) + y = y[seq_len + pred_step - 1:] # make it fit with x sequences + + self.x[i] = torch.from_numpy(x) + self.y[i] = torch.from_numpy(y) + sizes[i] = y.shape[0] + + if sizes[i] <= 0: + raise ValueError(f"Error with given seq_len: {seq_len} and pred_step: {pred_step} " + f"dataset can't return any data from file {f_path} (they are probably too big)") + + cumsizes = np.cumsum(sizes, dtype=np.int) + self.size = cumsizes[-1].item() + self.idx_file_map = np.searchsorted(cumsizes, np.arange(self.size) + 1).astype(np.int) + self.idx_file_idx_map = np.zeros(self.size).astype(np.int) + + self.idx_file_idx_map[0] = 0 + cnt = 1 + for i in range(1, len(self.idx_file_map)): + if self.idx_file_map[i] != self.idx_file_map[i-1]: + cnt = 0 + self.idx_file_idx_map[i] = cnt + cnt += 1 + + def __push_cols(self, cols: np.array, push_idx: List[int], d: int) -> np.array: + """ + pushes cols of indexes in push_idx by value d, + other cols get cutted by value d + """ + + if d == 0: + return cols + + new_cols = [None] * len(cols) + + for i, _ in enumerate(cols): + if i in push_idx: + new_cols[i] = cols[i][d:] + else: + new_cols[i] = cols[i][:-d] + + return new_cols + + def set_transforms(self, transforms): + self.transforms = transforms + + def get_uniform_dist_y_sampler(self) -> WeightedRandomSampler: + """ + returns instance of torch.utils.data.WeightedRandomSampler + with weights so that after applying it to as sampler in + pytorch dataloader, with train_dataset output mapping + will have almost uniform distribution + """ + + train_ds = self.get_train_dataset() + loader = DataLoader(train_ds, batch_size=len(train_ds)) + unique, indexes, occurs = np.unique( + next(iter(loader))[1], # get all "y" from train_dataset (outputs) + return_counts=True, + return_inverse=True, + axis=0 + ) + weights = 1 / (occurs[indexes] * unique.shape[0]) + return WeightedRandomSampler(weights, self.size, True) + + def __getitem(self, idx) -> torch.tensor: + """ + TODO update this docstr + returns tuple (seq, pred) where: + seq is torch.tensor of shape (seq_len, 3) + pred is torch.tensor of shape (6,) + """ + + f = self.idx_file_map[idx] + f_idx = self.idx_file_idx_map[idx] + x = self.x[f][f_idx:f_idx + self.seq_len] + y = self.y[f][f_idx] + + if self.transforms is not None: + x = self.transforms(x) + + return (x, y) + + def __get_dataset(self, start, size): + return SequenceForecastMultiDistributionDataset( + start, + size, + self.__getitem + ) + + def get_train_dataset(self) -> SequenceForecastMultiDistributionDataset: + return self.__get_dataset(0, self.__train_size(self.size)) + + def get_test_dataset(self) -> SequenceForecastMultiDistributionDataset: + return self.__get_dataset( + self.__train_size(self.size), + self.__test_size(self.size) + ) + + +class FCRdatasetFactory(SequenceForecastMultiDistributionDatasetFactory): + """ + FCR dataset factory + """ + + def __init__( + self, + seq_len: int, + pred_step: int, + transforms=None, + ): + from .data import DATA_FILES + + super().__init__( + seq_len=seq_len, + pred_step=pred_step, + transforms=transforms, + files=DATA_FILES, + x_y_split=3, + usecols=range(1, 10), + x_predictions_cols=range(1, 3) + ) + diff --git a/FCRdataLoader/tests/__init__.py b/FCRdataLoader/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/FCRdataLoader/tests/seq1pred0_test.py b/FCRdataLoader/tests/seq1pred0_test.py new file mode 100644 index 0000000000000000000000000000000000000000..9617f122fce425c1af313ed60f55525a7dfae4fd --- /dev/null +++ b/FCRdataLoader/tests/seq1pred0_test.py @@ -0,0 +1,35 @@ +import unittest +from fcrdataloader.dataset import SequenceForecastMultiDistributionDatasetFactory as SFMDDF +from torch.utils.data import DataLoader +from pathlib import Path +import os + + +TEST_FILE_DIR = 'tests/test_data' + + +class FCRCorrectDataTest(unittest.TestCase): + + def test_seq1_pred0(self): + factory = SFMDDF( + seq_len=1, + pred_step=0, + files=list(Path(TEST_FILE_DIR).glob("*")), + x_y_split=3, + usecols=range(1, 10), + x_predictions_cols=range(1, 3) + ) + train_set = factory.get_train_dataset() + loader = DataLoader(train_set, batch_size=1) + + for row, (x, y) in enumerate(loader): + x = x.reshape(-1) + y = y.reshape(-1) + for col in range(0, 3): + self.assertEqual(x[col], float(f"{row}.{col}")) + for col in range(0, 6): + self.assertEqual(y[col], float(f"{row}.{col + 3}")) + + +if __name__ == '__main__': + unittest.main() diff --git a/FCRdataLoader/tests/test_data/fcrdata10.csv b/FCRdataLoader/tests/test_data/fcrdata10.csv new file mode 100644 index 0000000000000000000000000000000000000000..285e8c51912490eda900e397d8e7a482ef2ccb58 --- /dev/null +++ b/FCRdataLoader/tests/test_data/fcrdata10.csv @@ -0,0 +1,12 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-11-27T12:44:35.519,10.0,10.1,10.2,10.3,10.4,10.5,10.6,10.7,10.8,10.9 +2020-11-27T12:44:35.519,11.0,11.1,11.2,11.3,11.4,11.5,11.6,11.7,11.8,11.9 +2020-11-27T12:44:35.519,12.0,12.1,12.2,12.3,12.4,12.5,12.6,12.7,12.8,12.9 +2020-11-27T12:44:35.519,13.0,13.1,13.2,13.3,13.4,13.5,13.6,13.7,13.8,13.9 +2020-11-27T12:44:35.519,14.0,14.1,14.2,14.3,14.4,14.5,14.6,14.7,14.8,14.9 +2020-11-27T12:44:35.519,15.0,15.1,15.2,15.3,15.4,15.5,15.6,15.7,15.8,15.9 +2020-11-27T12:44:35.519,16.0,16.1,16.2,16.3,16.4,16.5,16.6,16.7,16.8,16.9 +2020-11-27T12:44:35.519,17.0,17.1,17.2,17.3,17.4,17.5,17.6,17.7,17.8,17.9 +2020-11-27T12:44:35.519,18.0,18.1,18.2,18.3,18.4,18.5,18.6,18.7,18.8,18.9 +2020-11-27T12:44:35.519,18.0,18.1,18.2,18.3,18.4,18.5,18.6,18.7,18.8,18.9 +2020-11-27T12:44:35.519,20.0,20.1,20.2,20.3,20.4,20.5,20.6,20.7,20.8,20.9 diff --git a/FCRdataLoader/tests/test_data/fcrdata2.csv b/FCRdataLoader/tests/test_data/fcrdata2.csv new file mode 100644 index 0000000000000000000000000000000000000000..7ef612e27ced21478353fae1abd914a9cedb7d36 --- /dev/null +++ b/FCRdataLoader/tests/test_data/fcrdata2.csv @@ -0,0 +1,11 @@ +timestamp,AvgResponseTime,AvgResponseTimePrediction,split,cardinality_Component_LB,provider_Component_LB,AppCardinality,cardinality_Component_DB,provider_Component_App,provider_Component_DB +2020-11-27T12:44:35.519,0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9 +2020-11-27T12:44:35.519,1.0,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8,1.9 +2020-11-27T12:44:35.519,2.0,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8,2.9 +2020-11-27T12:44:35.519,3.0,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8,3.9 +2020-11-27T12:44:35.519,4.0,4.1,4.2,4.3,4.4,4.5,4.6,4.7,4.8,4.9 +2020-11-27T12:44:35.519,5.0,5.1,5.2,5.3,5.4,5.5,5.6,5.7,5.8,5.9 +2020-11-27T12:44:35.519,6.0,6.1,6.2,6.3,6.4,6.5,6.6,6.7,6.8,6.9 +2020-11-27T12:44:35.519,7.0,7.1,7.2,7.3,7.4,7.5,7.6,7.7,7.8,7.9 +2020-11-27T12:44:35.519,8.0,8.1,8.2,8.3,8.4,8.5,8.6,8.7,8.8,8.9 +2020-11-27T12:44:35.519,9.0,9.1,9.2,9.3,9.4,9.5,9.6,9.7,9.8,9.9 diff --git a/FCRgendata/Pipfile b/FCRgendata/Pipfile new file mode 100644 index 0000000000000000000000000000000000000000..782f7149ef567caa7b1ee1de9732aabf767cd9d7 --- /dev/null +++ b/FCRgendata/Pipfile @@ -0,0 +1,17 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +jsonschema = "*" +setuptools = "*" +pipenv-setup = "*" +numpy = "*" +lxml = "*" +progress = "*" + +[requires] +python_version = "3.8" diff --git a/FCRgendata/Pipfile.lock b/FCRgendata/Pipfile.lock new file mode 100644 index 0000000000000000000000000000000000000000..590980cc33abd688501c9561c51955c46ae9334a --- /dev/null +++ b/FCRgendata/Pipfile.lock @@ -0,0 +1,431 @@ +{ + "_meta": { + "hash": { + "sha256": "3ceafb48a7bc77840c0a82fa99004106b69e316ec11f6ba956adb538a5b1bd4c" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.3.0" + }, + "black": { + "hashes": [ + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" + ], + "markers": "python_version >= '3.6'", + "version": "==19.10b0" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "cerberus": { + "hashes": [ + "sha256:302e6694f206dd85cb63f13fd5025b31ab6d38c99c50c6d769f8fa0b0f299589" + ], + "version": "==1.3.2" + }, + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.4" + }, + "distlib": { + "hashes": [ + "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", + "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" + ], + "version": "==0.3.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "jsonschema": { + "hashes": [ + "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", + "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" + ], + "index": "pypi", + "version": "==3.2.0" + }, + "lxml": { + "hashes": [ + "sha256:0448576c148c129594d890265b1a83b9cd76fd1f0a6a04620753d9a6bcfd0a4d", + "sha256:127f76864468d6630e1b453d3ffbbd04b024c674f55cf0a30dc2595137892d37", + "sha256:1471cee35eba321827d7d53d104e7b8c593ea3ad376aa2df89533ce8e1b24a01", + "sha256:2363c35637d2d9d6f26f60a208819e7eafc4305ce39dc1d5005eccc4593331c2", + "sha256:2e5cc908fe43fe1aa299e58046ad66981131a66aea3129aac7770c37f590a644", + "sha256:2e6fd1b8acd005bd71e6c94f30c055594bbd0aa02ef51a22bbfa961ab63b2d75", + "sha256:366cb750140f221523fa062d641393092813b81e15d0e25d9f7c6025f910ee80", + "sha256:42ebca24ba2a21065fb546f3e6bd0c58c3fe9ac298f3a320147029a4850f51a2", + "sha256:4e751e77006da34643ab782e4a5cc21ea7b755551db202bc4d3a423b307db780", + "sha256:4fb85c447e288df535b17ebdebf0ec1cf3a3f1a8eba7e79169f4f37af43c6b98", + "sha256:50c348995b47b5a4e330362cf39fc503b4a43b14a91c34c83b955e1805c8e308", + "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf", + "sha256:535f067002b0fd1a4e5296a8f1bf88193080ff992a195e66964ef2a6cfec5388", + "sha256:5be4a2e212bb6aa045e37f7d48e3e1e4b6fd259882ed5a00786f82e8c37ce77d", + "sha256:60a20bfc3bd234d54d49c388950195d23a5583d4108e1a1d47c9eef8d8c042b3", + "sha256:648914abafe67f11be7d93c1a546068f8eff3c5fa938e1f94509e4a5d682b2d8", + "sha256:681d75e1a38a69f1e64ab82fe4b1ed3fd758717bed735fb9aeaa124143f051af", + "sha256:68a5d77e440df94011214b7db907ec8f19e439507a70c958f750c18d88f995d2", + "sha256:69a63f83e88138ab7642d8f61418cf3180a4d8cd13995df87725cb8b893e950e", + "sha256:6e4183800f16f3679076dfa8abf2db3083919d7e30764a069fb66b2b9eff9939", + "sha256:6fd8d5903c2e53f49e99359b063df27fdf7acb89a52b6a12494208bf61345a03", + "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d", + "sha256:7a7669ff50f41225ca5d6ee0a1ec8413f3a0d8aa2b109f86d540887b7ec0d72a", + "sha256:7e9eac1e526386df7c70ef253b792a0a12dd86d833b1d329e038c7a235dfceb5", + "sha256:7ee8af0b9f7de635c61cdd5b8534b76c52cd03536f29f51151b377f76e214a1a", + "sha256:8246f30ca34dc712ab07e51dc34fea883c00b7ccb0e614651e49da2c49a30711", + "sha256:8c88b599e226994ad4db29d93bc149aa1aff3dc3a4355dd5757569ba78632bdf", + "sha256:923963e989ffbceaa210ac37afc9b906acebe945d2723e9679b643513837b089", + "sha256:94d55bd03d8671686e3f012577d9caa5421a07286dd351dfef64791cf7c6c505", + "sha256:97db258793d193c7b62d4e2586c6ed98d51086e93f9a3af2b2034af01450a74b", + "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f", + "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc", + "sha256:d8d3d4713f0c28bdc6c806a278d998546e8efc3498949e3ace6e117462ac0a5e", + "sha256:e0bfe9bb028974a481410432dbe1b182e8191d5d40382e5b8ff39cdd2e5c5931", + "sha256:f4822c0660c3754f1a41a655e37cb4dbbc9be3d35b125a37fab6f82d47674ebc", + "sha256:f83d281bb2a6217cd806f4cf0ddded436790e66f393e124dfe9731f6b3fb9afe", + "sha256:fc37870d6716b137e80d19241d0e2cff7a7643b925dfa49b4c8ebd1295eb506e" + ], + "index": "pypi", + "version": "==4.6.2" + }, + "numpy": { + "hashes": [ + "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db", + "sha256:09c12096d843b90eafd01ea1b3307e78ddd47a55855ad402b157b6c4862197ce", + "sha256:13d166f77d6dc02c0a73c1101dd87fdf01339febec1030bd810dcd53fff3b0f1", + "sha256:141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512", + "sha256:16c1b388cc31a9baa06d91a19366fb99ddbe1c7b205293ed072211ee5bac1ed2", + "sha256:18bed2bcb39e3f758296584337966e68d2d5ba6aab7e038688ad53c8f889f757", + "sha256:1aeef46a13e51931c0b1cf8ae1168b4a55ecd282e6688fdb0a948cc5a1d5afb9", + "sha256:27d3f3b9e3406579a8af3a9f262f5339005dd25e0ecf3cf1559ff8a49ed5cbf2", + "sha256:2a2740aa9733d2e5b2dfb33639d98a64c3b0f24765fed86b0fd2aec07f6a0a08", + "sha256:4377e10b874e653fe96985c05feed2225c912e328c8a26541f7fc600fb9c637b", + "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb", + "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc", + "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac", + "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83", + "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36", + "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387", + "sha256:8cac8790a6b1ddf88640a9267ee67b1aee7a57dfa2d2dd33999d080bc8ee3a0f", + "sha256:8ece138c3a16db8c1ad38f52eb32be6086cc72f403150a79336eb2045723a1ad", + "sha256:9eeb7d1d04b117ac0d38719915ae169aa6b61fca227b0b7d198d43728f0c879c", + "sha256:a09f98011236a419ee3f49cedc9ef27d7a1651df07810ae430a6b06576e0b414", + "sha256:a5d897c14513590a85774180be713f692df6fa8ecf6483e561a6d47309566f37", + "sha256:ad6f2ff5b1989a4899bf89800a671d71b1612e5ff40866d1f4d8bcf48d4e5764", + "sha256:c42c4b73121caf0ed6cd795512c9c09c52a7287b04d105d112068c1736d7c753", + "sha256:cb1017eec5257e9ac6209ac172058c430e834d5d2bc21961dceeb79d111e5909", + "sha256:d6c7bb82883680e168b55b49c70af29b84b84abb161cbac2800e8fcb6f2109b6", + "sha256:e452dc66e08a4ce642a961f134814258a082832c78c90351b75c41ad16f79f63", + "sha256:e5b6ed0f0b42317050c88022349d994fe72bfe35f5908617512cd8c8ef9da2a9", + "sha256:e9b30d4bd69498fc0c3fe9db5f62fffbb06b8eb9321f92cc970f2969be5e3949", + "sha256:ec149b90019852266fec2341ce1db513b843e496d5a8e8cdb5ced1923a92faab", + "sha256:edb01671b3caae1ca00881686003d16c2209e07b7ef8b7639f1867852b948f7c", + "sha256:f0d3929fe88ee1c155129ecd82f981b8856c5d97bcb0d5f23e9b4242e79d1de3", + "sha256:f29454410db6ef8126c83bd3c968d143304633d45dc57b51252afbd79d700893", + "sha256:fe45becb4c2f72a0907c1d0246ea6449fe7a9e2293bb0e11c4e9a32bb0930a15", + "sha256:fedbd128668ead37f33917820b704784aff695e0019309ad446a6d0b065b57e4" + ], + "index": "pypi", + "version": "==1.19.4" + }, + "orderedmultidict": { + "hashes": [ + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" + ], + "version": "==1.0.1" + }, + "packaging": { + "hashes": [ + "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858", + "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.8" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, + "pep517": { + "hashes": [ + "sha256:3985b91ebf576883efe5fa501f42a16de2607684f3797ddba7202b71b7d0da51", + "sha256:aeb78601f2d1aa461960b43add204cc7955667687fbcf9cdb5170f00556f117f" + ], + "version": "==0.9.1" + }, + "pip-shims": { + "hashes": [ + "sha256:05b00ade9d1e686a98bb656dd9b0608a933897283dc21913fad6ea5409ff7e91", + "sha256:16ca9f87485667b16b978b68a1aae4f9cc082c0fa018aed28567f9f34a590569" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.5.3" + }, + "pipenv-setup": { + "hashes": [ + "sha256:8a439aff7b16e18d7e07702c9186fc5fe86156679eace90e10c2578a43bd7af1", + "sha256:e1bfd55c1152024e762f1c17f6189fcb073166509e7c0228870f7ea160355648" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pipfile": { + "hashes": [ + "sha256:f7d9f15de8b660986557eb3cc5391aa1a16207ac41bc378d03f414762d36c984" + ], + "version": "==0.0.2" + }, + "plette": { + "extras": [ + "validation" + ], + "hashes": [ + "sha256:46402c03e36d6eadddad2a5125990e322dd74f98160c8f2dcd832b2291858a26", + "sha256:d6c9b96981b347bddd333910b753b6091a2c1eb2ef85bb373b4a67c9d91dca16" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.3" + }, + "progress": { + "hashes": [ + "sha256:69ecedd1d1bbe71bf6313d88d1e6c4d2957b7f1d4f71312c211257f7dae64372" + ], + "index": "pypi", + "version": "==1.5" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "pyrsistent": { + "hashes": [ + "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" + ], + "markers": "python_version >= '3.5'", + "version": "==0.17.3" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.1" + }, + "regex": { + "hashes": [ + "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", + "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", + "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", + "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", + "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", + "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", + "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", + "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", + "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", + "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", + "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", + "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", + "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", + "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", + "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", + "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", + "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", + "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", + "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", + "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", + "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", + "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", + "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", + "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", + "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", + "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", + "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", + "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", + "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", + "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", + "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", + "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", + "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", + "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", + "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", + "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", + "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", + "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", + "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", + "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", + "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + ], + "version": "==2020.11.13" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.25.1" + }, + "requirementslib": { + "hashes": [ + "sha256:50d20f27e4515a2393695b0d886219598302163438ae054253147b2bad9b4a44", + "sha256:9c1e8666ca4512724cdd1739adcc7df19ec7ad2ed21f0e748f9631ad6b54f321" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.5.16" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "tomlkit": { + "hashes": [ + "sha256:6babbd33b17d5c9691896b0e68159215a9387ebfa938aa3ac42f4a4beeb2b831", + "sha256:ac57f29693fab3e309ea789252fcce3061e19110085aa31af5446ca749325618" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.7.0" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072", + "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298", + "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "version": "==1.4.1" + }, + "urllib3": { + "hashes": [ + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.2" + }, + "vistir": { + "hashes": [ + "sha256:a37079cdbd85d31a41cdd18457fe521e15ec08b255811e81aa061fd5f48a20fb", + "sha256:eff1d19ef50c703a329ed294e5ec0b0fbb35b96c1b3ee6dcdb266dddbe1e935a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.2" + }, + "wheel": { + "hashes": [ + "sha256:78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", + "sha256:e11eefd162658ea59a60a0f6c7d493a7190ea4b9a85e335b33489d9f17e0245e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.36.2" + } + }, + "develop": {} +} diff --git a/FCRgendata/config.json b/FCRgendata/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c9a0b27e5e4f46bcd246883af6151463600002b7 --- /dev/null +++ b/FCRgendata/config.json @@ -0,0 +1,17 @@ +{ + "request": { + "applicationId": "FCRwithDLMSApp", + "camelModelFilePath" : "/home/szysad/mimuw/3rok/ZPP/FCR-problems/FRC-dane-treningowe/FCR-model.xmi", + "cpProblemFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/FRC-dane-treningowe/FCR-CP.xmi", + "nodeCandidatesFilePath": "/home/szysad/mimuw/3rok/ZPP/FCR-problems/FRC-dane-treningowe/FCR-NodeCandidates", + "watermark": { + "user": "mrozanska", + "system": "UI", + "date": "2017-11-23T16: 41: 41+0000", + "uuid": "fb6280ec-1ab8-11e7-93ae-92361f002671" + } + }, + "cpSolverHost": "localhost:8080", + "outpath": "generated_data.csv", + "AvgResponseTimeTableFilePath": "/home/szysad/mimuw/3rok/ZPP/time-series-data/time-series-data/secure-document​/deployment-reconfiguration-range-2-to-2/2020-10-30 to 2020-10-30/V 1.0 - raw data/AvgResponseTimeTable.csv" +} \ No newline at end of file diff --git a/FCRgendata/readme.md b/FCRgendata/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..8a0f296e2d6216d3d181ced82c3d62b79a786f0f --- /dev/null +++ b/FCRgendata/readme.md @@ -0,0 +1,48 @@ +# Training data generator for FCR app +# How to install dependencies + - make sure you have python3 installed + - if you don't have pipenv already installed run: +```sh +$ pip install pipenv +``` + - now run in directory with `Pipfile` +```sh +$ pipenv install +``` +# How to run +### 1. Create config.json + - create `config.json` + - `request` object field must be same as one send to cp-solver to `\constraintProblemSolutionFromFile` endpoint + - `cpSolverHost` string field must represent, web address to working cp-solver instance + - `outpath` string field must represent, path for file with solution to be created + - `AvgResponseTimeTableFilePath` string field must be path to file with FCR time series + - `predictionsFilePath` string field must be path to file with FCR avgResponseTime predictions + +example of `config.json`: +```json +{ + "request": { + "applicationId": "FCRwithDLMSApp", + "camelModelFilePath" : "/home/FCR-data/FCR-model.xmi", + "cpProblemFilePath": "/home/FCR-data/FCR-CP.xmi", + "nodeCandidatesFilePath": "/home/FCR-data/FCR-NodeCandidates", + "watermark": { + "user": "mrozanska", + "system": "UI", + "date": "2017-11-23T16: 41: 41+0000", + "uuid": "fb6280ec-1ab8-11e7-93ae-92361f002671" + } + }, + "cpSolverHost": "localhost:2137", + "outpath": "generated_data.csv", + "AvgResponseTimeTableFilePath": "/home/FCR-time-series/AvgResponseTimeTable.csv", + "predictionsFilePath": "/home/FCR-time-series/predictions1.csv" +} +``` + +### 2. Run script + - in directory with `Pipfile` run: +```sh +$ pipenv shell +$ python -m src.main +``` \ No newline at end of file diff --git a/FCRgendata/setup.py b/FCRgendata/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..30f974ba34fc689fff62865eb33c7db482ab8b82 --- /dev/null +++ b/FCRgendata/setup.py @@ -0,0 +1,197 @@ +"""A setuptools based setup module. +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +Modified by Madoshakalaka@Github (dependency links added) +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +from os import path + +# io.open is needed for projects that support Python 2.7 +# It ensures open() defaults to text mode with universal newlines, +# and accepts an argument to specify the text encoding +# Python 3 only projects can skip this import +from io import open + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, "README.md"), encoding="utf-8") as f: + long_description = f.read() + +# Arguments marked as "Required" below must be included for upload to PyPI. +# Fields marked as "Optional" may be commented out. + +setup( + # This is the name of your project. The first time you publish this + # package, this name will be registered for you. It will determine how + # users can install this project, e.g.: + # + # $ pip install sampleproject + # + # And where it will live on PyPI: https://pypi.org/project/sampleproject/ + # + # There are some restrictions on what makes a valid project name + # specification here: + # https://packaging.python.org/specifications/core-metadata/#name + name="FCRgenData", # Required + # Versions should comply with PEP 440: + # https://www.python.org/dev/peps/pep-0440/ + # + # For a discussion on single-sourcing the version across setup.py and the + # project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version="0.0.0", # Required + # This is a one-line description or tagline of what your project does. This + # corresponds to the "Summary" metadata field: + # https://packaging.python.org/specifications/core-metadata/#summary + #description="A sample Python project", # Optional + # This is an optional longer description of your project that represents + # the body of text which users will see when they visit PyPI. + # + # Often, this is the same as your README, so you can just read it in from + # that file directly (as we have already done above) + # + # This field corresponds to the "Description" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-optional + #long_description=long_description, # Optional + # Denotes that our long_description is in Markdown; valid values are + # text/plain, text/x-rst, and text/markdown + # + # Optional if long_description is written in reStructuredText (rst) but + # required for plain-text or Markdown; if unspecified, "applications should + # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and + # fall back to text/plain if it is not valid rst" (see link below) + # + # This field corresponds to the "Description-Content-Type" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional + #long_description_content_type="text/markdown", # Optional (see note above) + # This should be a valid link to your project's main homepage. + # + # This field corresponds to the "Home-Page" metadata field: + # https://packaging.python.org/specifications/core-metadata/#home-page-optional + #url="https://github.com/pypa/sampleproject", # Optional + # This should be your name or the name of the organization which owns the + # project. + #author="The Python Packaging Authority", # Optional + # This should be a valid email address corresponding to the author listed + # above. + #author_email="pypa-dev@googlegroups.com", # Optional + # Classifiers help users find your project by categorizing it. + # + # For a list of valid classifiers, see https://pypi.org/classifiers/ + classifiers=[ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 3 - Alpha", + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + # Pick your license as you wish + "License :: OSI Approved :: MIT License", + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + # These classifiers are *not* checked by 'pip install'. See instead + # 'python_requires' below. + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], + # This field adds keywords for your project which will appear on the + # project page. What does your project relate to? + # + # Note that this is a string of words separated by whitespace, not a list. + #keywords="sample setuptools development", # Optional + #keywords="sample setuptools development", # Optional + # You can just specify package directories manually here if your project is + # simple. Or you can use find_packages(). + # + # Alternatively, if you just want to distribute a single Python file, use + # the `py_modules` argument instead as follows, which will expect a file + # called `my_module.py` to exist: + # + # py_modules=["my_module"], + # + packages=find_packages(), # Required + # Specify which Python versions you support. In contrast to the + # 'Programming Language' classifiers above, 'pip install' will check this + # and refuse to install the project if the version does not match. If you + # do not support Python 2, you can simplify this to '>=3.5' or similar, see + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4", + # This field lists other packages that your project depends on to run. + # Any package you put here will be installed by pip when your project is + # installed, so they must be valid existing projects. + # + # For an analysis of "install_requires" vs pip's requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + "appdirs==1.4.4", + "attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "black==19.10b0; python_version >= '3.6'", + "cached-property==1.5.2", + "cerberus==1.3.2", + "certifi==2020.12.5", + "chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "distlib==0.3.1", + "idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "jsonschema==3.2.0", + "orderedmultidict==1.0.1", + "packaging==20.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pathspec==0.8.1", + "pep517==0.9.1", + "pip-shims==0.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "pipenv-setup==3.1.1", + "pipfile==0.0.2", + "plette[validation]==0.2.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pyrsistent==0.17.3; python_version >= '3.5'", + "python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "regex==2020.11.13", + "requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "requirementslib==1.5.16; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "tomlkit==0.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "typed-ast==1.4.1", + "urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "vistir==0.5.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "wheel==0.36.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + ], # Optional + # List additional groups of dependencies here (e.g. development + # dependencies). Users will be able to install these using the "extras" + # syntax, for example: + # + # $ pip install sampleproject[dev] + # + # Similar to `install_requires` above, these must be valid existing + # projects. + extras_require={"dev": []}, # Optional + # If there are data files included in your packages that need to be + # installed, specify them here. + # + # Sometimes you’ll want to use packages that are properly arranged with + # setuptools, but are not published to PyPI. In those cases, you can specify + # a list of one or more dependency_links URLs where the package can + # be downloaded, along with some additional hints, and setuptools + # will find and install the package correctly. + # see https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi + # + dependency_links=[], + package_dir={'': 'src'}, + entry_points={ + 'console_scripts': [ + 'fcrgendata = FCRGenData.__main__:run', + ], + }, + +) diff --git a/FCRgendata/src/FCRGenData/__init__.py b/FCRgendata/src/FCRGenData/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/FCRgendata/src/FCRGenData/__main__.py b/FCRgendata/src/FCRGenData/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..a81e509310b6561758608ab824e846ab99dc94c4 --- /dev/null +++ b/FCRgendata/src/FCRGenData/__main__.py @@ -0,0 +1,82 @@ +""" Generates data using CP-solver and FCR time series for network training """ + +import csv +import logging +import sys +from contextlib import ExitStack +from pathlib import Path + +import numpy as np +import numpy.lib.recfunctions as rfn +from progress.bar import IncrementalBar as IBar + +from .solv_summoner import CPSolverSolutionSummoner, CPSolverSolutionSummonerError +from .validate_config import validate_config + +logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) +CONFIG_PATH: Path = None + + +def run(): + if len(sys.argv) != 2: + logging.critical( + f"received {len(sys.argv) - 1} arguments, required one: config file path.") + sys.exit(1) + else: + CONFIG_PATH = Path(sys.argv[1]) + + j_conf = validate_config(CONFIG_PATH) + avg_rsp_times = np.genfromtxt( # shape (n, ) + j_conf['AvgResponseTimeTableFilePath'], + delimiter=',', + dtype=np.dtype([('avg_rsp_time', np.float64), ('datetime', 'datetime64[s]')]), + skip_header=1, + usecols=(6, 1) + ) + + avg_rsp_prediction = np.genfromtxt( # shape (n, ) + j_conf['predictionsFilePath'], + delimiter=',', + dtype=np.dtype([('avg_rsp_time_pred', np.float64), ('datetime', 'datetime64[s]'), ('split', np.int)]), + skip_header=1, + usecols=(1, 3, 4), + converters={4: lambda split: {b"train": 0, b"val": 1, b"test": -1}[split]} + ) + + joined = rfn.join_by('datetime', avg_rsp_times, avg_rsp_prediction, jointype='inner', usemask=False) + assert len(joined) == len(avg_rsp_prediction) + # joined is structured array of tuples with dtype + # [('datetime', ' List[str]: + """Filters column with names including any phrase from keys list.""" + return list(filter(lambda itername: any(map(lambda keyname: keyname in itername, keys)), column)) + + +class RawCSVReader(IRawDataProvider): + """CSV data reader implementation.""" + __path: Path + __delimiter: str + + __arr: pd.DataFrame + + __lines: List[str] + + __column_names: Tuple[str] + __column_types: Dict[str, type] + + def __init__(self, + path: Union[Path, str], + delimiter: str = ',', + timestamp_column_name: Optional[str] = None): + super().__init__(timestamp_column_name=timestamp_column_name) + + """ + Args: + path: (Union[Path, str]) path to the csv file. + delimiter: (str) csv file felimiter. + timestamp_column_name: (str) timestamp column name (if unset, it will be guessed). + """ + # Check args + assert path is not None, 'Unset path!' + if isinstance(path, str): + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f'File {path} not found') + self.__path = path + self.__delimiter = delimiter + + # Peek headers + with open(self.__path, newline='') as file: + assert csv.Sniffer().has_header(file.readline()), "CSV file has no header" + file.seek(0) + columns = file.readline().strip('\n').split(delimiter) + + time_columns = _match_columns(columns, TIME_COLUMN_NAMES) + + timestamp_columns = _match_columns(columns, self._timestamp_column_names) + + assert len(timestamp_columns) == 1, f'Cannot specify timestamp column, found column names: {columns}' + timestamp_column_name = timestamp_columns[0] + + # Read from file = be aware of high memory usage + self.__arr = pd.read_csv( + path, + parse_dates=time_columns + ) + assert self.__arr[timestamp_column_name].is_monotonic_increasing, 'Timestamps in column are not increasing' + assert self.__arr[timestamp_column_name].is_unique, 'Found >=2 equal timestamps' + + self.__column_types = {} + + for key, name in self.__arr.dtypes.apply(lambda x: x.name).to_dict().items(): + if name == 'object': + new_value = str + elif name.startswith('datetime'): + new_value = dt.datetime + elif name.startswith('float'): + new_value = float + elif name.startswith('int'): + new_value = int + elif name == 'bool': + new_value = bool + else: + raise NotImplementedError("Unknown datatype format") + self.__column_types[key] = new_value + + @property + def column_names(self) -> Tuple[str]: + """Column names""" + return tuple(self.__arr.columns) + + @property + def columns(self) -> Dict[str, type]: + """Column names mapping to it's types""" + return self.__column_types + + @staticmethod + def _convert_to_pytype(value): + if isinstance(value, pd._libs.tslibs.timestamps.Timestamp): + return value.to_pydatetime() + else: + return value + + def reader(self) -> Generator[Iterable[type], None, None]: + """Returns iterator over rows. + + Yields: + Iterable[type]: values in the next row (order is the same as in column_names). + """ + for index, row in self.__arr.iterrows(): + values = row.values.tolist() + values = map(RawCSVReader._convert_to_pytype, values) + yield list(values) + + def reader_annotated(self) -> Generator[Dict[str, type], None, None]: + """Returns dict iterator over rows. + + Yields: + Dicttype[str, ]: name of the columns mapping to its values in the current row. + """ + for index, row in self.__arr.iterrows(): + mapped_values = dict(row.to_dict()) + for key, val in mapped_values.items(): + mapped_values[key] = RawCSVReader._convert_to_pytype(val) + yield mapped_values diff --git a/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py b/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..def04a150a6c0a984dd4677fe13b75fee5649810 --- /dev/null +++ b/FCRgendata/src/FCRGenData/rawDataReader/raw_data_reader.py @@ -0,0 +1,47 @@ +import abc +from typing import Tuple, Dict, Generator, Iterable, Optional, List + +TIME_COLUMN_NAMES = ['time', 'date', 'timestamp', 'datetime'] +TIMESTAMP_KEY_COLUMN_NAMES = ['timestamp', 'datetime'] + + +class IRawDataProvider(abc.ABC): + """Data provider interface specification. Supplied with + time-series data generators, which provides data measured in + increasing, non repetitive timestamps. Time between timestamps *may* + vary.""" + + _timestamp_column_names: List[str] + + def __init__(self, + timestamp_column_name: Optional[str] = None): + if timestamp_column_name: + self._timestamp_column_names = [timestamp_column_name] + else: + self._timestamp_column_names = TIMESTAMP_KEY_COLUMN_NAMES + + @property + @abc.abstractmethod + def column_names(self) -> Tuple[str]: + """Names of the variable columns""" + pass + + @property + @abc.abstractmethod + def columns(self) -> Dict[str, type]: + """Names of the variable columns with mapping to it's types""" + pass + + @abc.abstractmethod + def reader(self) -> Generator[Iterable[type], None, None]: + """Generator over raw data, provides rows of data in increasing order (by timestamp) + Returns list with values (order same as in column_names). + """ + pass + + @abc.abstractmethod + def reader_annotated(self) -> Generator[Dict[str, type], None, None]: + """Generator over raw data, provides rows of data in increasing order (by timestamp). + Returns dict with column names mapping to current values. + """ + pass diff --git a/FCRgendata/src/FCRGenData/solv_summoner.py b/FCRgendata/src/FCRGenData/solv_summoner.py new file mode 100644 index 0000000000000000000000000000000000000000..0378848a2222dc87b62e40f5294093f5861cd6fc --- /dev/null +++ b/FCRgendata/src/FCRGenData/solv_summoner.py @@ -0,0 +1,60 @@ +from pathlib import Path +from lxml import etree +from tempfile import NamedTemporaryFile +from contextlib import ContextDecorator +from typing import List +from functools import lru_cache +import logging +import requests +import json +import os + + +CACHE_SIZE = 2**10 + + +class CPSolverSolutionSummonerError(Exception): + pass + + +class CPSolverSolutionSummoner(ContextDecorator): + """ calls cp-solver for solution, and manages additional resources """ + + def __init__(self, config): + self.solver_host = config['cpSolverHost'] + self.solution_file = f"{config['request']['camelModelFilePath'][:-4]}-solution.xmi" + self.FCRcpxml = etree.parse(config['request']['cpProblemFilePath']) + self.avg_rsp_time_xelem, = self.FCRcpxml.find( + "cpMetrics[@id='AvgResponseTime']") + self.req = config['request'] + + def __enter__(self): + self.tempfile = NamedTemporaryFile() + self.req['cpProblemFilePath'] = self.tempfile.name + return self + + def __exit__(self, *exc): + self.tempfile.close() + if os.path.exists(self.solution_file): + os.remove(self.solution_file) + return False + + @lru_cache(maxsize=CACHE_SIZE) + def get_solution(self, avg_rsp_time: float) -> List[int]: + """ returns solution as list of int """ + + self.avg_rsp_time_xelem.set('value', str(avg_rsp_time)) + self.FCRcpxml.write(self.tempfile.name, + xml_declaration=True, encoding="ASCII") + r = requests.post(f'http://{self.solver_host}/constraintProblemSolutionFromFile', + data=json.dumps(self.req), + headers={'Content-Type': 'application/json'} + ) + + if r.status_code != 200: + raise Exception("cp-solver returned non 200 status code") + + sol_xelem = etree.parse(self.solution_file).find("solution") + if sol_xelem is None: + raise CPSolverSolutionSummonerError(f"cp-solver didn't return solution for avg_rsp_time = {avg_rsp_time}") + return [int(v.get('value', 0)) for v in sol_xelem.iterfind(".//value")] diff --git a/FCRgendata/src/FCRGenData/validate_config.py b/FCRgendata/src/FCRGenData/validate_config.py new file mode 100644 index 0000000000000000000000000000000000000000..09724281b25cf967bb647a3da208683a5a8f055d --- /dev/null +++ b/FCRgendata/src/FCRGenData/validate_config.py @@ -0,0 +1,82 @@ +"""config validation module + +Defines validate_config function responsible for validating path to config file, +it's structure and content +""" + +from pathlib import Path +import logging +import json +import jsonschema +import sys + + +config_schema = { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "applicationId": {"type": "string"}, + "camelModelFilePath": {"type": "filepath"}, + "cpProblemFilePath": {"type": "filepath"}, + "nodeCandidatesFilePath": {"type": "filepath"}, + "watermark": { + "type": "object", + "properties": { + "user": {"type": "string"}, + "system": {"type": "string"}, + "date": {"type": "string"}, + "uuid": {"type": "string"}, + }, + "minProperties": 4 + } + }, + "minProperties": 5 + }, + "AvgResponseTimeTableFilePath": {"type": "filepath"}, + "predictionsFilePath": {"type": "filepath"}, + "cpSolverHost": {"type": "string"}, # for example "localhost:8080" + "outpath": {"type": "creatablepath"}, + }, + "required": ["request", "AvgResponseTimeTableFilePath", "cpSolverHost", "outpath", "predictionsFilePath"] +} + +# create two custom types: "filepath" (path to exsisting file) +# and "creatablepath" (path in which new file can be created) +type_checker = jsonschema.Draft3Validator.TYPE_CHECKER.redefine_many({ + "filepath": lambda checker, path: Path(path).is_file(), + "creatablepath": lambda checker, path: Path(path).parent.is_dir() +}) +customValidator = jsonschema.validators.extend( + jsonschema.Draft3Validator, type_checker=type_checker) +validator = customValidator(schema=config_schema) + + +def validate_config(conf_p: Path): + """ validates given config path, config path structure and it content, + returns validated json object + """ + + if not conf_p.is_file(): + logging.critical(f"given path: {conf_p} doesn't lead to file") + sys.exit(1) + + j_conf = None + with conf_p.open() as f: + try: + j_conf = json.load(f) + except json.decoder.JSONDecodeError as err: + logging.critical(f"error decoding json config: {err}") + sys.exit(1) + + try: + validator.validate(j_conf) + except jsonschema.exceptions.ValidationError as e: + logging.critical(f'json validation error: {e.message}') + sys.exit(1) + except jsonschema.exceptions.SchemaError as e: + logging.critical(f'json schema error: {e.message}') + sys.exit(1) + + return j_conf diff --git a/FCRgendata/tests/data_reader/data_reader_template.py b/FCRgendata/tests/data_reader/data_reader_template.py new file mode 100644 index 0000000000000000000000000000000000000000..6b9b501a22eb7e9121915c6d7aa036cb8b0b5ebb --- /dev/null +++ b/FCRgendata/tests/data_reader/data_reader_template.py @@ -0,0 +1,124 @@ +import abc +import csv +import datetime as dt +import io +from typing import Callable, Optional + +import pytest + +from FCRGenData.rawDataReader import IRawDataProvider + + +class RawDataProviderTestTemplate(abc.ABC): + @abc.abstractmethod + @pytest.yield_fixture + def reader_factory(self) -> Callable[[str, Optional[str]], IRawDataProvider]: + pass + + def test_import_basic(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() + dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + # unknown timestamp column + with pytest.raises(AssertionError) as exc: + reader = reader_factory(s.read()) + + assert 'Cannot specify timestamp column' in exc.value.args[0] + + reader_factory(s.read(), timestamp_column_name='time') + + def test_time_desc(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() - dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + # unknown timestamp column + with pytest.raises(AssertionError) as exc: + reader = reader_factory(s.read(), timestamp_column_name='time') + + assert 'Timestamps in column are not increasing' in exc.value.args[0] + + def test_time_eq(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + t = dt.datetime.now() + for i in range(5): + row = [t, i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + # unknown timestamp column + with pytest.raises(AssertionError) as exc: + reader = reader_factory(s.read(), timestamp_column_name='time') + + assert 'Found >=2 equal timestamps' in exc.value.args[0] + + def test_column_getters(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() + dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + + reader: IRawDataProvider = reader_factory(s.read(), timestamp_column_name='time') + + assert len(reader.column_names) == len(header) + for reader_col, header_col in zip(reader.column_names, header): + assert reader_col == header_col + + col_types = reader.columns + + # check types + assert col_types['time'] == dt.datetime + for i in range(1, len(header)): + col = header[i] + assert col_types[col].__name__ == col + + def test_read(self, reader_factory): + header = ['time', 'int', 'str', 'float', 'bool'] + content = [] + for i in range(5): + row = [dt.datetime.now() + dt.timedelta(days=i), i, str(i) + 'pln', i / 10, i % 2 == 0] + assert len(row) == len(header) + content.append(row) + + s = io.StringIO() + csv.writer(s, delimiter=',').writerows([header] + content) + + s.seek(0) + + reader: IRawDataProvider = reader_factory(s.read(), timestamp_column_name='time') + + row_gen = reader.reader() + for row, grow in zip(content, row_gen): + assert row == grow + + dict_gen = reader.reader_annotated() + for row, row_dict in zip(content, dict_gen): + for col_name, value in zip(header, row): + assert row_dict[col_name] == value diff --git a/FCRgendata/tests/data_reader/test_csv_reader.py b/FCRgendata/tests/data_reader/test_csv_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..6f6436a9dc6ec8b207a42d7993119d170ff088ca --- /dev/null +++ b/FCRgendata/tests/data_reader/test_csv_reader.py @@ -0,0 +1,28 @@ +import tempfile +from typing import Callable + +import pytest + +from FCRGenData.rawDataReader import RawCSVReader +from FCRgendata.tests.data_reader.data_reader_template import RawDataProviderTestTemplate + + +class TestCSVReader(RawDataProviderTestTemplate): + + @pytest.yield_fixture + def empty_reader(self) -> RawCSVReader: + tfile = tempfile.NamedTemporaryFile(suffix='.csv') + yield RawCSVReader(tfile.name) + tfile.close() + + @pytest.yield_fixture + def reader_factory(self) -> Callable[[str], RawCSVReader]: + tfile = tempfile.NamedTemporaryFile(suffix='.csv', mode='w') + path = tfile.name + + def _write(s, *args, **kwargs): + tfile.write(s) + tfile.flush() + return RawCSVReader(path, *args, **kwargs) + + yield _write diff --git a/FCRtraining/README.md b/FCRtraining/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1abe27b1106196dfc5e2295c66bef1a546754b8c --- /dev/null +++ b/FCRtraining/README.md @@ -0,0 +1,110 @@ +# pytorch lightning training setup for FCR app +# How to install dependencies + - make sure you have latest version of pytorch and pytorch-lightning installed + +

+ +# Contents +## FCRtraining.networks.LitFCRtestBase.BaseTestEncoder + - subclass of pytorch_lightning.LightningModule + - implements testing and training metrics logging + - use it as base of your LightningModule class + - details about LightningModule: https://pytorch-lightning.readthedocs.io/en/stable/lightning_module.html + +

+ +## Metrics logging + - after running experiment directory `lightning_logs` should appear + - you can inspect logs with `tensorboard` by running: +```sh +$ tensorboard --logdir= +``` + +# Example of BaseTestEncoder usage +```Python +from FCRdataLoader.fcrdataloader.dataset import FCRtrainDataSet, FCRtestDataSet +from torch.utils.data import DataLoader +from torch.optim.lr_scheduler import ReduceLROnPlateau +from .LitFCRtestBase import BaseTestEncoder +import torch.nn as nn +import torch + + +HIDDEN_SIZE = 40 +BATCH_SIZE = 32 +SEQ_LEN = 10 +HORIZON = 0 +LR = 0.01 + +FEATURES = 3 +OUTPUT = 6 + + +class Encoder(BaseTestEncoder): + def __init__( + self, + features=FEATURES, + output=OUTPUT, + learning_rate=LR, + batch_size=BATCH_SIZE, + seq_len=SEQ_LEN, + horizon=HORIZON, + hidden_size=HIDDEN_SIZE, + + ): + super(Encoder, self).__init__() + + self.seq_len = seq_len + self.horizon = horizon + self.batch_size = batch_size + + self.criterion = nn.MSELoss() + self.lr = learning_rate + + self.lstm = nn.LSTM(features, hidden_size, num_layers=2, + bidirectional=True, batch_first=True) + self.fc = nn.Linear(hidden_size * 2, output) + + def forward(self, x): + out, _ = self.lstm(x) + # out: (batch, features, hidden_size * directions) + out = out[:, -1, :] + # out: (batch, hidden_size * directions) + out = self.fc(out) + return out + + def training_step(self, batch, batch_idx): + x, y = batch + prediction = self(x) + loss = self.criterion(prediction, y) + + self.log('train_loss', loss, on_step=False, on_epoch=True) + + return loss + + def val_dataloader(self): + return self.test_dataloader() + + def train_dataloader(self): + train_data = FCRtrainDataSet(self.seq_len, self.horizon) + loader = DataLoader(train_data, batch_size=self.batch_size, + num_workers=4)#, sampler=train_data.get_weighted_rnd_sampler()) + return loader + + def test_dataloader(self): + test_data = FCRtestDataSet(self.seq_len, self.horizon) + loader = DataLoader(test_data, batch_size=self.batch_size, + num_workers=4)#, sampler=test_data.get_weighted_rnd_sampler()) + return loader + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.lr) + scheduler = ReduceLROnPlateau( + optimizer, 'min', patience=10, verbose=True) + return { + 'optimizer': optimizer, + 'lr_scheduler': scheduler, + 'monitor': 'train_loss' + } + +``` \ No newline at end of file diff --git a/FCRtraining/metrics/RowAccuracy.py b/FCRtraining/metrics/RowAccuracy.py new file mode 100644 index 0000000000000000000000000000000000000000..e2bb4680b010bef587f751e73ecafddeec33208d --- /dev/null +++ b/FCRtraining/metrics/RowAccuracy.py @@ -0,0 +1,26 @@ +from pytorch_lightning.metrics import Metric +import torch + + +class RowAccuracy(Metric): + """ + Represents Accuracy of matches in dim=1 in givens tensors + + implemented as in: + https://pytorch-lightning.readthedocs.io/en/latest/metrics.html + """ + + def __init__(self, dist_sync_on_step=False): + super().__init__(dist_sync_on_step=dist_sync_on_step) + + self.add_state("correct", default=torch.tensor(0), dist_reduce_fx="sum") + self.add_state("total", default=torch.tensor(0), dist_reduce_fx="sum") + + def update(self, preds: torch.Tensor, target: torch.Tensor): + preds, target = preds, target + assert preds.shape == target.shape + self.correct += (preds == target).all(dim=1).sum() # count all row matches + self.total += target.shape[0] # add batch size + + def compute(self): + return self.correct.float() / self.total diff --git a/FCRtraining/networks/LitFCRtestBase.py b/FCRtraining/networks/LitFCRtestBase.py new file mode 100644 index 0000000000000000000000000000000000000000..64100eec4a74f22876db8fc56ca33056dc8444af --- /dev/null +++ b/FCRtraining/networks/LitFCRtestBase.py @@ -0,0 +1,92 @@ +from ..metrics.RowAccuracy import RowAccuracy +import pytorch_lightning as pl +import torch.nn as nn +import torch + + +class BaseTestEncoder(pl.LightningModule): + """ + abstract base class for LightningModule, + implements validation and test loops including logging + subclass must implement criterion as loss function + """ + + def __init__(self): + ''' + creates train, and test metrics + TODO: add metric from smoteR paper + ''' + + super(BaseTestEncoder, self).__init__() + + # train metrics + self.train_rounded_accuracy = RowAccuracy() + self.train_rounded_mse = pl.metrics.MeanSquaredError() + self.train_rounded_mae = pl.metrics.MeanAbsoluteError() + self.train_mse = pl.metrics.MeanSquaredError() + self.train_mae = pl.metrics.MeanAbsoluteError() + + # test metrics + self.test_rounded_accuracy = RowAccuracy() + self.test_rounded_mse = pl.metrics.MeanSquaredError() + self.test_rounded_mae = pl.metrics.MeanAbsoluteError() + self.test_mse = pl.metrics.MeanSquaredError() + self.test_mae = pl.metrics.MeanAbsoluteError() + + def validation_step(self, batch, batch_nb): + """ + predicts y, and calculates loss in training + """ + + x, y = batch + preds = self(x) + loss = self.criterion(preds, y) # might not be necessary + + return {'loss': loss, 'preds': preds, 'target': y} + + def validation_step_end(self, outputs): + ''' + update and log validation metrics + ''' + + rounded_preds = torch.round(outputs['preds']) + self.train_rounded_accuracy(rounded_preds, outputs['target']) + self.train_rounded_mse(rounded_preds, outputs['target']) + self.train_rounded_mae(rounded_preds, outputs['target']) + self.train_mse(outputs['preds'], outputs['target']) + self.train_mae(outputs['preds'], outputs['target']) + + self.log('train_rounded_accuracy', self.train_rounded_accuracy, on_step=False, on_epoch=True) + self.log('train_rounded_mse', self.train_rounded_mse, on_step=False, on_epoch=True) + self.log('train_rounded_mae', self.train_rounded_mae, on_step=False, on_epoch=True) + self.log('train_mse', self.train_mse, on_step=False, on_epoch=True) + self.log('train_mae', self.train_mae, on_step=False, on_epoch=True) + + def test_step(self, batch, batch_idx): + """ + predicts y, and calculates loss in testing + """ + + x, y = batch + preds = self(x) + loss = self.criterion(preds, y) # might not be necessary + + return {'loss': loss, 'preds': preds, 'target': y} + + def test_step_end(self, outputs): + ''' + update and log test metrics + ''' + + rounded_preds = torch.round(outputs['preds']) + self.test_rounded_accuracy(rounded_preds, outputs['target']) + self.test_rounded_mse(rounded_preds, outputs['target']) + self.test_rounded_mae(rounded_preds, outputs['target']) + self.test_mse(outputs['preds'], outputs['target']) + self.test_mae(outputs['preds'], outputs['target']) + + self.log('test_rounded_accuracy', self.test_rounded_accuracy, on_step=False, on_epoch=True) + self.log('test_rounded_mse', self.test_rounded_mse, on_step=False, on_epoch=True) + self.log('test_rounded_mae', self.test_rounded_mae, on_step=False, on_epoch=True) + self.log('test_mse', self.test_mse, on_step=False, on_epoch=True) + self.log('test_mae', self.test_mae, on_step=False, on_epoch=True) diff --git a/FCRtraining/pathloader.py b/FCRtraining/pathloader.py new file mode 100644 index 0000000000000000000000000000000000000000..ce2bfecf30a81763814514dbcb2a5911cb16eeee --- /dev/null +++ b/FCRtraining/pathloader.py @@ -0,0 +1,3 @@ +import sys +import os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) \ No newline at end of file