From 77b5ec93a7e05526b64ebd04fa556c2e01b3cd67 Mon Sep 17 00:00:00 2001 From: Nadesh Seen Date: Thu, 13 Jul 2023 02:04:55 -0400 Subject: [PATCH] updated new version 12-07-2023 --- controlplane/composer.log | 137 ++++++++++++ controlplane/dag_specifications/dag2.json | 2 +- .../text_sentiment_analysis_dag.json | 49 ++++ controlplane/dagit.log | 209 ++++++++++++++++++ controlplane/orchestrator.py | 197 +++++++++++++++-- controlplane/trigger_gateway.py | 90 +++++--- .../sentiment_analysis_trigger.json | 6 + .../calculate_text_sentiment/Dockerfile | 11 + .../calculate_text_sentiment/build.,txt | 4 + .../calculate_text_sentiment/buildAndPush.sh | 24 ++ .../calculate_text_sentiment/params.json | 7 + .../calculate_text_sentiment/requirements.txt | 5 + .../calculate_text_sentiment/sentiment.py | 32 +++ .../create_sentiment_report/Dockerfile | 11 + .../create_sentiment_report/build.,txt | 4 + .../create_sentiment_report/buildAndPush.sh | 24 ++ .../create_sentiment_report/params.json | 7 + .../sentiment_report.py | 41 ++++ function_modules/url_to_summary/Dockerfile | 11 + function_modules/url_to_summary/build.,txt | 4 + .../url_to_summary/buildAndPush.sh | 24 ++ .../url_to_summary/data_processing.py | 43 ++++ function_modules/url_to_summary/params.json | 3 + .../url_to_summary/requirements.txt | 5 + load_test.py | 41 ++++ 25 files changed, 937 insertions(+), 54 deletions(-) create mode 100644 controlplane/composer.log create mode 100644 controlplane/dag_specifications/text_sentiment_analysis_dag.json create mode 100644 controlplane/dagit.log create mode 100644 controlplane/trigger_specifications/sentiment_analysis_trigger.json create mode 100644 function_modules/calculate_text_sentiment/Dockerfile create mode 100644 function_modules/calculate_text_sentiment/build.,txt create mode 100755 function_modules/calculate_text_sentiment/buildAndPush.sh create mode 100755 function_modules/calculate_text_sentiment/params.json create mode 100644 function_modules/calculate_text_sentiment/requirements.txt create mode 100644 function_modules/calculate_text_sentiment/sentiment.py create mode 100644 function_modules/create_sentiment_report/Dockerfile create mode 100644 function_modules/create_sentiment_report/build.,txt create mode 100755 function_modules/create_sentiment_report/buildAndPush.sh create mode 100755 function_modules/create_sentiment_report/params.json create mode 100644 function_modules/create_sentiment_report/sentiment_report.py create mode 100644 function_modules/url_to_summary/Dockerfile create mode 100644 function_modules/url_to_summary/build.,txt create mode 100755 function_modules/url_to_summary/buildAndPush.sh create mode 100644 function_modules/url_to_summary/data_processing.py create mode 100755 function_modules/url_to_summary/params.json create mode 100644 function_modules/url_to_summary/requirements.txt create mode 100644 load_test.py diff --git a/controlplane/composer.log b/controlplane/composer.log new file mode 100644 index 0000000..c87a9fb --- /dev/null +++ b/controlplane/composer.log @@ -0,0 +1,137 @@ +Datetime Activation ID Kind Start Duration Status Entity + +2023-07-10 09:07:21 87d58732b2ab4a4c958732b2abfa4cb9 nodejs:10 warm 11ms success guest/toonify:0.0.1 +2023-07-10 09:07:16 b8a5db9e63304f6ba5db9e63304f6b00 blackbox cold 4.973s success guest/encode-function:0.0.2 +2023-07-10 09:07:09 e73772ddcbb74425b772ddcbb7b4252a nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 09:07:03 24286a51cc634d28a86a51cc638d28fe blackbox cold 6.366s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:06:55 c6a9bf400a314e13a9bf400a313e1382 nodejs:10 warm 12ms success guest/toonify:0.0.1 +2023-07-10 09:06:48 6971c4fe88db45b7b1c4fe88dba5b7d8 blackbox cold 6.767s success guest/decode-function:0.0.2 +2023-07-10 09:06:40 2a604a2855074152a04a285507d1524c nodejs:10 cold 340ms success guest/toonify:0.0.1 +2023-07-10 09:06:39 4d4ddc0b779a40f88ddc0b779ab0f810 sequence warm 42.063s success guest/toonify:0.0.1 + +2023-07-10 09:15:41 11138ff9894c48d4938ff9894cc8d4dc nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:15:36 66b2361aa1dc4c4fb2361aa1dc0c4f40 blackbox warm 4.261s success guest/encode-function:0.0.2 +2023-07-10 09:15:36 66055fbc569c4e60855fbc569c2e6047 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:15:29 79ea6cf3b4a84909aa6cf3b4a869093f blackbox warm 6.566s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:15:29 68072402f8d946dd872402f8d9c6ddc7 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 09:15:23 581c570a3b314daa9c570a3b31adaa9d blackbox warm 5.579s success guest/decode-function:0.0.2 +2023-07-10 09:15:23 b2dddd1a638d46ab9ddd1a638d16ab9a nodejs:10 warm 14ms success guest/toonify:0.0.1 +2023-07-10 09:15:23 278416ec92ea4dfc8416ec92ea5dfca4 sequence warm 17.199s success guest/toonify:0.0.1 + +2023-07-10 09:29:08 a171a7fb536a4d91b1a7fb536a9d9132 nodejs:10 warm 4ms success guest/toonify:0.0.1 +2023-07-10 09:29:04 c820c1079dea42fea0c1079deac2febf blackbox warm 4.541s success guest/encode-function:0.0.2 +2023-07-10 09:29:04 af0e6aeae9d34b758e6aeae9d3eb75f6 nodejs:10 warm 4ms success guest/toonify:0.0.1 +2023-07-10 09:28:58 f3456f54c20943c2856f54c20933c24d blackbox warm 6.267s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:28:58 b42230657f09460fa230657f09360f70 nodejs:10 warm 6ms success guest/toonify:0.0.1 +2023-07-10 09:28:53 b914b41a39e5448994b41a39e56489c4 blackbox warm 4.625s success guest/decode-function:0.0.2 +2023-07-10 09:28:53 d73b3fc9e8f249dcbb3fc9e8f209dc15 nodejs:10 cold 116ms success guest/toonify:0.0.1 +2023-07-10 09:28:53 43612625494344caa12625494364ca80 sequence warm 15.979s success guest/toonify:0.0.1 + +2023-07-10 09:43:36 1f7951021e7744f0b951021e7714f045 nodejs:10 warm 6ms success guest/toonify:0.0.1 +2023-07-10 09:43:32 4e065cd4f8e34a07865cd4f8e3da07bc blackbox warm 4.556s success guest/encode-function:0.0.2 +2023-07-10 09:43:32 88f1f4fea3c14e80b1f4fea3c19e80e6 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:43:25 9a68ca50381847fda8ca50381807fdba blackbox warm 6.537s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:43:25 1876750d46024e12b6750d46028e12dd nodejs:10 warm 9ms success guest/toonify:0.0.1 +2023-07-10 09:43:20 bf9d6aa2ee3044c29d6aa2ee3054c22b blackbox warm 5.568s success guest/decode-function:0.0.2 +2023-07-10 09:43:19 ddb6a116b4f54e27b6a116b4f59e27cf nodejs:10 cold 692ms success guest/toonify:0.0.1 +2023-07-10 09:43:18 4199190f449d49a099190f449d79a08c sequence warm 18.362s success guest/toonify:0.0.1 + +2023-07-10 09:44:51 f3a9061cfc7c41a6a9061cfc7c41a6b7 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:44:47 76e4a1dd6f124b5aa4a1dd6f12bb5a22 blackbox warm 4.109s success guest/encode-function:0.0.2 +2023-07-10 09:44:47 59a33ce2902144bda33ce2902174bdcf nodejs:10 warm 13ms success guest/toonify:0.0.1 +2023-07-10 09:44:39 ff0f517c025e41878f517c025ee18797 blackbox warm 7.202s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:44:39 7e7e1f7611d04128be1f7611d08128ba nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:44:34 533fbbdb22dd4084bfbbdb22dd10844e blackbox warm 5.27s success guest/decode-function:0.0.2 +2023-07-10 09:44:34 f1dbb9197db643b89bb9197db683b853 nodejs:10 warm 12ms success guest/toonify:0.0.1 +2023-07-10 09:44:34 1dd36563edab4bcc936563edabdbccec sequence warm 16.841s success guest/toonify:0.0.1 + +2023-07-10 09:46:20 edd70c7156a9449e970c7156a9649e64 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:46:16 f20bffdd54414b538bffdd54413b530c blackbox warm 4.223s success guest/encode-function:0.0.2 +2023-07-10 09:46:16 69edad4b7a5044a9adad4b7a5064a9cb nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:46:09 3c9a8c612879421a9a8c612879f21a60 blackbox warm 6.54s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:46:09 ba5a0e5650a147c89a0e5650a157c843 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 09:46:03 f35e763999fa48b99e763999fa48b9ee blackbox warm 6.096s success guest/decode-function:0.0.2 +2023-07-10 09:46:03 de2b1b2a09bf4576ab1b2a09bf45766f nodejs:10 warm 14ms success guest/toonify:0.0.1 +2023-07-10 09:46:03 82a0e1b6e4cc49aaa0e1b6e4cc39aad9 sequence warm 17.1s success guest/toonify:0.0.1 + +2023-07-10 09:50:22 f0b89d51bdd84f63b89d51bdd8cf6317 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:50:18 93a9155161b74422a9155161b7a42272 blackbox warm 4.202s success guest/encode-function:0.0.2 +2023-07-10 09:50:18 4383ed2cd2b2437383ed2cd2b2737356 nodejs:10 warm 6ms success guest/toonify:0.0.1 +2023-07-10 09:50:11 90684dfeea964797a84dfeea967797df blackbox warm 6.486s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:50:11 8dd67827757d4ded967827757d2ded69 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 09:50:06 2b2bcac2611f443dabcac2611f543d32 blackbox warm 5.385s success guest/decode-function:0.0.2 +2023-07-10 09:50:06 54b8cd959fdd4954b8cd959fdd49541e nodejs:10 warm 14ms success guest/toonify:0.0.1 +2023-07-10 09:50:06 004001f12b7145678001f12b71856753 sequence warm 16.337s success guest/toonify:0.0.1 + +2023-07-10 09:53:35 024ae0477680490f8ae0477680f90f61 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:53:31 9dae7e94d38e45bcae7e94d38e35bca7 blackbox warm 4.082s success guest/encode-function:0.0.2 +2023-07-10 09:53:31 4559bfdcbfa24e0f99bfdcbfa27e0fff nodejs:10 warm 12ms success guest/toonify:0.0.1 +2023-07-10 09:53:25 d7f0e15380854f5fb0e1538085cf5f26 blackbox warm 6.2s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:53:25 775e68084995442b9e68084995242b37 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 09:53:19 419417f266b342ef9417f266b362ef69 blackbox warm 5.488s success guest/decode-function:0.0.2 +2023-07-10 09:53:19 64aba8e496254baeaba8e49625dbaee9 nodejs:10 warm 16ms success guest/toonify:0.0.1 +2023-07-10 09:53:19 3bf5ae2ea0534e28b5ae2ea053ae2830 sequence warm 15.993s success guest/toonify:0.0.1 + +2023-07-10 09:55:37 3ae60bc95b694ba0a60bc95b69dba005 nodejs:10 warm 6ms success guest/toonify:0.0.1 +2023-07-10 09:55:33 ece254b11d024708a254b11d02d708c1 blackbox warm 3.82s success guest/encode-function:0.0.2 +2023-07-10 09:55:33 2e300028268e4515b00028268e2515d4 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 09:55:27 fc009fbbab9d4e5c809fbbab9d3e5c83 blackbox warm 6.258s success guest/image-bilateral-filter:0.0.2 +2023-07-10 09:55:26 354356de0d494e028356de0d496e02b6 nodejs:10 warm 11ms success guest/toonify:0.0.1 +2023-07-10 09:55:21 bda155ced40f4be2a155ced40f3be2f9 blackbox warm 5.391s success guest/decode-function:0.0.2 +2023-07-10 09:55:21 7dcec94b134d40138ec94b134d901384 nodejs:10 warm 15ms success guest/toonify:0.0.1 +2023-07-10 09:55:21 beedcbf69e984982adcbf69e989982c7 sequence warm 16.02s success guest/toonify:0.0.1 + +2023-07-10 10:02:13 a65b1ba54bb94da99b1ba54bb94da946 nodejs:10 warm 9ms success guest/toonify:0.0.1 +2023-07-10 10:02:08 cf7fd81cecf64da8bfd81cecf62da858 blackbox warm 4.731s success guest/encode-function:0.0.2 +2023-07-10 10:02:08 9ca421bcb201403ba421bcb201703b42 nodejs:10 warm 14ms success guest/toonify:0.0.1 +2023-07-10 10:02:02 4d81fcec4ad0420e81fcec4ad0b20ec3 blackbox warm 6.317s success guest/image-bilateral-filter:0.0.2 +2023-07-10 10:02:02 324b17979c0040eb8b17979c00e0ebf5 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 10:01:56 368db3f5dabf4b7c8db3f5dabffb7cd8 blackbox warm 5.452s success guest/decode-function:0.0.2 +2023-07-10 10:01:56 31b474d32d2440d5b474d32d24f0d596 nodejs:10 warm 17ms success guest/toonify:0.0.1 +2023-07-10 10:01:56 5c8da754786c48be8da754786cd8be93 sequence warm 16.89s success guest/toonify:0.0.1 + +2023-07-10 10:02:59 d0f4aab1b5f94595b4aab1b5f9c595d1 nodejs:10 warm 6ms success guest/toonify:0.0.1 +2023-07-10 10:02:55 e12a9877439842edaa9877439832edbb blackbox warm 3.585s success guest/encode-function:0.0.2 +2023-07-10 10:02:55 190ad2eeef9f4a3d8ad2eeef9f1a3d74 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 10:02:48 1574045c87b34661b4045c87b32661ee blackbox warm 6.883s success guest/image-bilateral-filter:0.0.2 +2023-07-10 10:02:48 47531ec13ea042e6931ec13ea0d2e6f3 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 10:02:43 439e8a21eefc42579e8a21eefcf25787 blackbox warm 5.462s success guest/decode-function:0.0.2 +2023-07-10 10:02:43 74a3dd7076ab4208a3dd7076ab2208bb nodejs:10 warm 10ms success guest/toonify:0.0.1 +2023-07-10 10:02:43 8237dab7860745bfb7dab7860775bf4e sequence warm 16.12s success guest/toonify:0.0.1 + +2023-07-10 10:04:00 50b8caa207eb495ab8caa207eb395ac2 nodejs:10 warm 9ms success guest/toonify:0.0.1 +2023-07-10 10:03:56 3b467cbefa544175867cbefa54017547 blackbox warm 4.257s success guest/encode-function:0.0.2 +2023-07-10 10:03:56 722d6186d3bd4468ad6186d3bdf4683f nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 10:03:48 6a37a4da5088466bb7a4da5088066b63 blackbox warm 7.694s success guest/image-bilateral-filter:0.0.2 +2023-07-10 10:03:48 a155e1136ea94c6095e1136ea98c6082 nodejs:10 warm 12ms success guest/toonify:0.0.1 +2023-07-10 10:03:43 f28f7a16552a42aa8f7a16552a42aa76 blackbox warm 5.09s success guest/decode-function:0.0.2 +2023-07-10 10:03:43 c9307967ff0642b7b07967ff0642b7cf nodejs:10 warm 9ms success guest/toonify:0.0.1 +2023-07-10 10:03:43 0586e20d6f0f43c586e20d6f0fa3c568 sequence warm 17.356s success guest/toonify:0.0.1 +2023-07-10 10:02:59 d0f4aab1b5f94595b4aab1b5f9c595d1 nodejs:10 warm 6ms success guest/toonify:0.0.1 + +2023-07-10 10:07:45 00ecfa9bf1a34354acfa9bf1a3a35440 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 10:07:41 714e2565cb9745578e2565cb97855781 blackbox warm 3.991s success guest/encode-function:0.0.2 +2023-07-10 10:07:41 231b7b1ecc274d5b9b7b1ecc27ed5bc2 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 10:07:35 89f8be2d67864db3b8be2d67866db329 blackbox warm 6.339s success guest/image-bilateral-filter:0.0.2 +2023-07-10 10:07:35 1edc7abee2bf4bc49c7abee2bfbbc4e5 nodejs:10 warm 10ms success guest/toonify:0.0.1 +2023-07-10 10:07:23 249a1dbf2b2d496c9a1dbf2b2d996ccb blackbox warm 11.457s success guest/decode-function:0.0.2 +2023-07-10 10:07:23 28c31eab1e334f07831eab1e339f070d nodejs:10 warm 16ms success guest/toonify:0.0.1 +2023-07-10 10:07:23 168c91d46e8141c58c91d46e81d1c5f4 sequence warm 22.026s success guest/toonify:0.0.1 + +2023-07-10 10:08:55 43a10a9e49614e5ba10a9e49619e5bd6 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 10:08:51 f26b26994a7f41d6ab26994a7fa1d6f9 blackbox warm 4.413s success guest/encode-function:0.0.2 +2023-07-10 10:08:51 cdcbf28cdd024c7e8bf28cdd027c7e0c nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 10:08:43 3b82e0423831461c82e0423831361ca2 blackbox warm 7.912s success guest/image-bilateral-filter:0.0.2 +2023-07-10 10:08:43 c258f285787e43d798f285787eb3d742 nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 10:08:38 107045e68897470fb045e68897870f25 blackbox warm 4.779s success guest/decode-function:0.0.2 +2023-07-10 10:08:38 3d4c0cdc8d224c0e8c0cdc8d22cc0e38 nodejs:10 warm 12ms success guest/toonify:0.0.1 +2023-07-10 10:08:38 2ced1b4026094f99ad1b4026099f994a sequence warm 17.309s success guest/toonify:0.0.1 + +2023-07-10 10:09:51 baf62a27fd7e4a7ab62a27fd7e9a7a6a nodejs:10 warm 6ms success guest/toonify:0.0.1 +2023-07-10 10:09:46 adb42379ec604e00b42379ec602e00d4 blackbox warm 4.218s success guest/encode-function:0.0.2 +2023-07-10 10:09:46 fdd87b30fad44fe3987b30fad4afe375 nodejs:10 warm 6ms success guest/toonify:0.0.1 +2023-07-10 10:09:40 0a3908237d2b498fb908237d2ba98f3e blackbox warm 6.122s success guest/image-bilateral-filter:0.0.2 +2023-07-10 10:09:40 7eacf5f175324012acf5f17532e012b5 nodejs:10 warm 7ms success guest/toonify:0.0.1 +2023-07-10 10:09:35 99e8fa7cd3164fbfa8fa7cd316bfbfc7 blackbox warm 5.1s success guest/decode-function:0.0.2 +2023-07-10 10:09:35 02cb2bd534824ff18b2bd53482cff1db nodejs:10 warm 8ms success guest/toonify:0.0.1 +2023-07-10 10:09:35 836780745a4b445fa780745a4b645f93 sequence warm 15.79s success guest/toonify:0.0.1 \ No newline at end of file diff --git a/controlplane/dag_specifications/dag2.json b/controlplane/dag_specifications/dag2.json index 82f75e0..8d6e0e5 100644 --- a/controlplane/dag_specifications/dag2.json +++ b/controlplane/dag_specifications/dag2.json @@ -33,7 +33,7 @@ "node_id": "encode-function", "properties": { - "label": "Cmobine Images to Video", + "label": "Combine Images to Video", "primitive": "serial", "condition":{}, "next": "", diff --git a/controlplane/dag_specifications/text_sentiment_analysis_dag.json b/controlplane/dag_specifications/text_sentiment_analysis_dag.json new file mode 100644 index 0000000..3bcfd24 --- /dev/null +++ b/controlplane/dag_specifications/text_sentiment_analysis_dag.json @@ -0,0 +1,49 @@ +{ + "name": "text-sentiment-analysis", + "dag": [ + { + "node_id": "fetch_sentences", + "properties": + { + "label": "Fetch Sentences", + "primitive": "serial", + "condition":{}, + "next": "calculate_sentiment", + "branch_1": "", + "branch_2": "", + "arguments": {}, + "outputs_from":[] + } + }, + { + "node_id": "calculate_sentiment", + "properties": + { + "label": "Calculate Sentiment Polarity", + "primitive": "serial", + "condition":{}, + "next": "create_sentiment_report", + "branch_1": "", + "branch_2": "", + "arguments": {}, + "outputs_from": ["fetch_sentences"] + } + }, + { + "node_id": "create_sentiment_report", + "properties": + { + "label": "Create a sentiment report for sentences", + "primitive": "serial", + "condition":{}, + "next": "", + "branch_1": "", + "branch_2": "", + "arguments": {}, + "outputs_from": ["fetch_sentences","calculate_sentiment"] + } + } + + ] + +} diff --git a/controlplane/dagit.log b/controlplane/dagit.log new file mode 100644 index 0000000..3ffa0ee --- /dev/null +++ b/controlplane/dagit.log @@ -0,0 +1,209 @@ +2023-07-09 15:51:06,729 - INFO - Function decode-function dequed from queue: +2023-07-09 15:51:06,729 - INFO - Function decode-function started execution +2023-07-09 15:51:23,927 - INFO - Function decode-function completed execution +2023-07-09 15:51:23,930 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-09 15:51:23,930 - INFO - Function image-bilateral-filter started execution +2023-07-09 15:51:50,527 - INFO - Function image-bilateral-filter completed execution +2023-07-09 15:51:50,529 - INFO - Function encode-function dequed from queue: +2023-07-09 15:51:50,529 - INFO - Function encode-function started execution +2023-07-09 15:52:06,452 - INFO - Function encode-function completed execution + +2023-07-10 09:14:32,019 - INFO - Function decode-function dequed from queue: +2023-07-10 09:14:32,019 - INFO - Function decode-function started execution +2023-07-10 09:14:37,826 - INFO - Function decode-function completed execution +2023-07-10 09:14:37,829 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:14:37,830 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:14:44,098 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:14:44,099 - INFO - Function encode-function dequed from queue: +2023-07-10 09:14:44,099 - INFO - Function encode-function started execution +2023-07-10 09:14:48,562 - INFO - Function encode-function completed execution + +2023-07-10 09:17:42,030 - INFO - Function decode-function dequed from queue: +2023-07-10 09:17:42,031 - INFO - Function decode-function started execution +2023-07-10 09:17:47,152 - INFO - Function decode-function completed execution +2023-07-10 09:17:47,155 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:17:47,156 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:17:54,471 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:17:54,472 - INFO - Function encode-function dequed from queue: +2023-07-10 09:17:54,472 - INFO - Function encode-function started execution +2023-07-10 09:17:59,897 - INFO - Function encode-function completed execution + +2023-07-10 09:27:10,941 - INFO - Function decode-function dequed from queue: +2023-07-10 09:27:10,952 - INFO - Function decode-function started execution +2023-07-10 09:27:15,823 - INFO - Function decode-function completed execution +2023-07-10 09:27:15,825 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:27:15,825 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:27:22,290 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:27:22,291 - INFO - Function encode-function dequed from queue: +2023-07-10 09:27:22,291 - INFO - Function encode-function started execution +2023-07-10 09:27:26,216 - INFO - Function encode-function completed execution + +2023-07-10 09:28:31,280 - INFO - Function decode-function dequed from queue: +2023-07-10 09:28:31,280 - INFO - Function decode-function started execution +2023-07-10 09:28:36,019 - INFO - Function decode-function completed execution +2023-07-10 09:28:36,020 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:28:36,021 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:28:42,363 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:28:42,365 - INFO - Function encode-function dequed from queue: +2023-07-10 09:28:42,366 - INFO - Function encode-function started execution +2023-07-10 09:28:46,095 - INFO - Function encode-function completed execution + +2023-07-10 09:31:47,633 - INFO - Function decode-function dequed from queue: +2023-07-10 09:31:47,633 - INFO - Function decode-function started execution +2023-07-10 09:31:53,248 - INFO - Function decode-function completed execution +2023-07-10 09:31:53,251 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:31:53,251 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:31:58,971 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:31:58,972 - INFO - Function encode-function dequed from queue: +2023-07-10 09:31:58,972 - INFO - Function encode-function started execution +2023-07-10 09:32:04,408 - INFO - Function encode-function completed execution + +2023-07-10 09:33:45,850 - INFO - Function decode-function dequed from queue: +2023-07-10 09:33:45,850 - INFO - Function decode-function started execution +2023-07-10 09:33:51,190 - INFO - Function decode-function completed execution +2023-07-10 09:33:51,192 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:33:51,192 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:33:57,826 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:33:57,827 - INFO - Function encode-function dequed from queue: +2023-07-10 09:33:57,827 - INFO - Function encode-function started execution +2023-07-10 09:34:02,892 - INFO - Function encode-function completed execution + +2023-07-10 09:34:43,087 - INFO - Function decode-function dequed from queue: +2023-07-10 09:34:43,087 - INFO - Function decode-function started execution +2023-07-10 09:34:48,983 - INFO - Function decode-function completed execution +2023-07-10 09:34:48,985 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:34:48,985 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:34:54,875 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:34:54,877 - INFO - Function encode-function dequed from queue: +2023-07-10 09:34:54,877 - INFO - Function encode-function started execution +2023-07-10 09:34:59,635 - INFO - Function encode-function completed execution + +2023-07-10 09:35:21,503 - INFO - Function decode-function dequed from queue: +2023-07-10 09:35:21,503 - INFO - Function decode-function started execution +2023-07-10 09:35:27,702 - INFO - Function decode-function completed execution +2023-07-10 09:35:27,704 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:35:27,704 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:35:34,777 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:35:34,778 - INFO - Function encode-function dequed from queue: +2023-07-10 09:35:34,778 - INFO - Function encode-function started execution +2023-07-10 09:35:40,218 - INFO - Function encode-function completed execution + +2023-07-10 09:36:28,894 - INFO - Function decode-function dequed from queue: +2023-07-10 09:36:28,894 - INFO - Function decode-function started execution +2023-07-10 09:36:34,917 - INFO - Function decode-function completed execution +2023-07-10 09:36:34,919 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:36:34,919 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:36:42,199 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:36:42,200 - INFO - Function encode-function dequed from queue: +2023-07-10 09:36:42,200 - INFO - Function encode-function started execution +2023-07-10 09:36:46,215 - INFO - Function encode-function completed execution + +2023-07-10 09:39:43,216 - INFO - Function decode-function dequed from queue: +2023-07-10 09:39:43,216 - INFO - Function decode-function started execution +2023-07-10 09:39:47,239 - INFO - Function decode-function completed execution +2023-07-10 09:39:47,241 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:39:47,241 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:39:50,417 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:39:50,418 - INFO - Function encode-function dequed from queue: +2023-07-10 09:39:50,418 - INFO - Function encode-function started execution +2023-07-10 09:39:53,025 - INFO - Function encode-function completed execution + +2023-07-10 09:39:53,343 - INFO - Function decode-function dequed from queue: +2023-07-10 09:39:53,344 - INFO - Function decode-function started execution +2023-07-10 09:39:57,592 - INFO - Function decode-function completed execution +2023-07-10 09:39:57,594 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:39:57,594 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:40:01,215 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:40:01,216 - INFO - Function encode-function dequed from queue: +2023-07-10 09:40:01,216 - INFO - Function encode-function started execution +2023-07-10 09:40:04,219 - INFO - Function encode-function completed execution + +2023-07-10 09:40:04,442 - INFO - Function decode-function dequed from queue: +2023-07-10 09:40:04,442 - INFO - Function decode-function started execution +2023-07-10 09:40:08,676 - INFO - Function decode-function completed execution +2023-07-10 09:40:08,678 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:40:08,678 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:40:11,983 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:40:11,985 - INFO - Function encode-function dequed from queue: +2023-07-10 09:40:11,985 - INFO - Function encode-function started execution +2023-07-10 09:40:14,752 - INFO - Function encode-function completed execution + +2023-07-10 09:40:14,965 - INFO - Function decode-function dequed from queue: +2023-07-10 09:40:14,965 - INFO - Function decode-function started execution +2023-07-10 09:40:19,271 - INFO - Function decode-function completed execution +2023-07-10 09:40:19,273 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:40:19,274 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:40:22,651 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:40:22,653 - INFO - Function encode-function dequed from queue: +2023-07-10 09:40:22,653 - INFO - Function encode-function started execution +2023-07-10 09:40:25,659 - INFO - Function encode-function completed execution + +2023-07-10 09:40:25,865 - INFO - Function decode-function dequed from queue: +2023-07-10 09:40:25,866 - INFO - Function decode-function started execution +2023-07-10 09:40:29,403 - INFO - Function decode-function completed execution +2023-07-10 09:40:29,405 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-10 09:40:29,405 - INFO - Function image-bilateral-filter started execution +2023-07-10 09:40:32,193 - INFO - Function image-bilateral-filter completed execution +2023-07-10 09:40:32,194 - INFO - Function encode-function dequed from queue: +2023-07-10 09:40:32,194 - INFO - Function encode-function started execution +2023-07-10 09:40:34,558 - INFO - Function encode-function completed execution + + +2023-07-11 01:20:19,599 - INFO - Function decode-function dequed from queue: +2023-07-11 01:20:19,599 - INFO - Json request for action decode-function is {'filename': 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4', 'parts': 10} +2023-07-11 01:20:19,599 - INFO - Function decode-function started execution +2023-07-11 01:20:31,596 - INFO - Function decode-function completed execution +2023-07-11 01:20:31,599 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-11 01:20:31,599 - INFO - Json request for action image-bilateral-filter is {'activation_id': 'e7bacc36986d43adbacc36986d53adfb', 'file_link': 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4', 'image_url_links': ['Image0.jpg', 'Image1.jpg', 'Image2.jpg', 'Image3.jpg', 'Image4.jpg', 'Image5.jpg', 'Image6.jpg', 'Image7.jpg', 'Image8.jpg', 'Image9.jpg'], 'parts': 10} +2023-07-11 01:20:31,600 - INFO - Function image-bilateral-filter started execution +2023-07-11 01:20:45,694 - INFO - Function image-bilateral-filter completed execution +2023-07-11 01:20:45,697 - INFO - Function encode-function dequed from queue: +2023-07-11 01:20:45,697 - INFO - Json request for action encode-function is {'activation_id': '8ce2c21f194e427ca2c21f194e327cf2', 'bilateral_filtered_image_links': ['bilateral_filtered_image_0.jpg', 'bilateral_filtered_image_1.jpg', 'bilateral_filtered_image_2.jpg', 'bilateral_filtered_image_3.jpg', 'bilateral_filtered_image_4.jpg', 'bilateral_filtered_image_5.jpg', 'bilateral_filtered_image_6.jpg', 'bilateral_filtered_image_7.jpg', 'bilateral_filtered_image_8.jpg', 'bilateral_filtered_image_9.jpg'], 'parts': 10} +2023-07-11 01:20:45,697 - INFO - Function encode-function started execution +2023-07-11 01:20:57,901 - INFO - Function encode-function completed execution + +2023-07-11 01:41:16,452 - INFO - Function decode-function dequed from queue: +2023-07-11 01:41:16,452 - INFO - Json request for action decode-function is {'filename': 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4', 'parts': 5} +2023-07-11 01:41:16,452 - INFO - Function decode-function started execution +2023-07-11 01:41:26,662 - INFO - Function decode-function completed execution +2023-07-11 01:41:26,664 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-11 01:41:26,664 - INFO - Json request for action image-bilateral-filter is {'activation_id': '09c36ff88f854558836ff88f85055802', 'file_link': 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4', 'image_url_links': ['Image0.jpg', 'Image1.jpg', 'Image2.jpg', 'Image3.jpg', 'Image4.jpg'], 'parts': 5} +2023-07-11 01:41:26,664 - INFO - Function image-bilateral-filter started execution +2023-07-11 01:41:40,647 - INFO - Function image-bilateral-filter completed execution +2023-07-11 01:41:40,649 - INFO - Function encode-function dequed from queue: +2023-07-11 01:41:40,649 - INFO - Json request for action encode-function is {'activation_id': '9d54f8655e2f46b594f8655e2ff6b5db', 'bilateral_filtered_image_links': ['bilateral_filtered_image_0.jpg', 'bilateral_filtered_image_1.jpg', 'bilateral_filtered_image_2.jpg', 'bilateral_filtered_image_3.jpg', 'bilateral_filtered_image_4.jpg'], 'parts': 5} +2023-07-11 01:41:40,649 - INFO - Function encode-function started execution +2023-07-11 01:41:51,986 - INFO - Function encode-function completed execution + +2023-07-11 02:21:51,515 - INFO - Function decode-function dequed from queue: +2023-07-11 02:21:51,516 - INFO - Function decode-function started execution +2023-07-11 02:22:04,238 - INFO - Function decode-function completed execution || Function ID : 53d371ed3f1f4edb9371ed3f1f9edbec +2023-07-11 02:22:04,241 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-11 02:22:04,241 - INFO - Function image-bilateral-filter started execution +2023-07-11 02:22:16,203 - INFO - Function image-bilateral-filter completed execution || Function ID : 0b4b888983014d928b88898301cd92b9 +2023-07-11 02:22:16,205 - INFO - Function encode-function dequed from queue: +2023-07-11 02:22:16,205 - INFO - Function encode-function started execution +2023-07-11 02:22:27,840 - INFO - Function encode-function completed execution || Function ID : 26f86408686448d2b864086864a8d271 + +2023-07-11 03:01:36,994 - INFO - Function decode-function dequed from queue: +2023-07-11 03:01:36,994 - INFO - Function decode-function started execution +2023-07-11 03:01:41,103 - INFO - Function decode-function completed execution || Function ID : c9b25f01ae1942e0b25f01ae1902e009 +2023-07-11 03:01:41,106 - INFO - Function image-bilateral-filter dequed from queue: +2023-07-11 03:01:41,107 - INFO - Function image-bilateral-filter started execution +2023-07-11 03:01:53,549 - INFO - Function image-bilateral-filter completed execution || Function ID : fd2d60379edd40b1ad60379edd90b139 +2023-07-11 03:01:53,551 - INFO - Function encode-function dequed from queue: +2023-07-11 03:01:53,552 - INFO - Function encode-function started execution +2023-07-11 03:02:04,103 - INFO - Function encode-function completed execution || Function ID : e3d63d9f4f9e4ae5963d9f4f9ebae524 +2023-07-11 03:02:04,105 - INFO - DAG Unique ID ed67c565-1788-4238-af03-6c96281a53a8 + +2023-07-11 06:00:34,380 - INFO - Function fetch_sentences dequed from queue: +2023-07-11 06:00:34,381 - INFO - Function fetch_sentences started execution +2023-07-11 06:00:37,186 - INFO - Function fetch_sentences completed execution || Function ID : 8a922842f6fd4a00922842f6fd7a00db +2023-07-11 06:00:37,189 - INFO - Function calculate_sentiment dequed from queue: +2023-07-11 06:00:37,190 - INFO - Function calculate_sentiment started execution +2023-07-11 06:00:37,407 - INFO - Function calculate_sentiment completed execution || Function ID : 0e8a567f24924b788a567f2492fb78e5 +2023-07-11 06:00:37,409 - INFO - Function create_sentiment_report dequed from queue: +2023-07-11 06:00:37,410 - INFO - Function create_sentiment_report started execution +2023-07-11 06:00:45,856 - INFO - Function create_sentiment_report completed execution || Function ID : 53fceeb33d994184bceeb33d99e18463 +2023-07-11 06:00:45,858 - INFO - DAG Unique ID 349946d3-bb6e-4efc-b2e6-ad62cd14d123 +2023-07-11 06:00:45,869 - INFO - 10.59.5.186 - - [11/Jul/2023 06:00:45] "POST /run/text_sentiment_analysis_trigger HTTP/1.1" 200 - diff --git a/controlplane/orchestrator.py b/controlplane/orchestrator.py index 9d72918..5de8f0f 100644 --- a/controlplane/orchestrator.py +++ b/controlplane/orchestrator.py @@ -4,32 +4,34 @@ import sys import requests import uuid import re +import datetime import subprocess import threading import queue import redis -from flask import current_app import pickle import json import os import time from requests.packages.urllib3.exceptions import InsecureRequestWarning -from flask import Flask, request,jsonify,send_file requests.packages.urllib3.disable_warnings(InsecureRequestWarning) import pymongo +import logging + +# Configure the logging settings +logging.basicConfig(filename='dagit.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + -# app = Flask(__name__) action_url_mappings = {} #Store action->url mappings action_properties_mapping = {} #Stores the action name and its corresponding properties responses = [] queue = [] -list_of_func_ids = [] dag_responses = [] +list_of_func_ids = [] function_responses = [] -x = 10 def preprocess(filename): @@ -127,19 +129,16 @@ def execute_action(action_name,request): def execute_dag(dag_name,request): - # print("------------------------------------DAG START-----------------------------------------------") - unique_id = uuid.uuid4() - print("DAG UNIQUE ID----------",unique_id) - dag_metadata={} - dag_metadata["dag_id"] = str(unique_id) - dag_metadata["dag_name"] = dag_name - list_of_func_ids = [] ######### Updates the list of action->url mapping ################### script_file = './actions.sh' subprocess.call(['bash', script_file]) ##################################################################### preprocess("action_url.txt") + + + list_of_func_ids = [] + ### Create in-memory redis storage ### redis_instace = create_redis_instance() ####################################### @@ -160,27 +159,34 @@ def execute_dag(dag_name,request): while(len(queue)!=0): flag=flag+1 action = queue.pop(0) - print("ACTION DEQUEUED FROM QUEUE : --->",action) + logging.info("Function {} dequed from queue:".format(action)) ########################################################## # HANDLE THE ACTION # ########################################################## if isinstance(action, str): # if(isinstance(action_properties_mapping[action]['arguments'],list)): # pass + json_data = action_properties_mapping[action]["arguments"] + # logging.info("Json request for action {} is {}".format(action,json_data)) url = action_url_mappings[action] + logging.info("Function {} started execution".format(action)) reply = requests.post(url = url,json=json_data,verify=False) + # print("Reply:",reply) + logging.info("Function {} completed execution || Function ID : {}".format(action,reply.json()["activation_id"])) + list_of_func_ids.append(reply.json()["activation_id"]) - # print("Line 292------------",reply.json()["activation_id"]) redis_instace.set(action+"-output",pickle.dumps(reply.json())) action_type = action_properties_mapping[action]["primitive"] + if(action_type=="condition"): branching_action = action_properties_mapping[action]["branch_1"] alternate_action = action_properties_mapping[action]["branch_2"] result=reply.json()["result"] condition_op = action_properties_mapping[action]["condition"]["operator"] if(condition_op=="equals"): + if(isinstance(action_properties_mapping[action]["condition"]["target"], str)): target = action_properties_mapping[action]["condition"]["target"] else: @@ -218,14 +224,156 @@ def execute_dag(dag_name,request): if(condition_op=="greater_than"): - pass + + if(isinstance(action_properties_mapping[action]["condition"]["target"], str)): + target = action_properties_mapping[action]["condition"]["target"] + else: + target=int(action_properties_mapping[action]["condition"]["target"]) + + if(result>target): + output_list = [] # List to store the output of actions whose outputs are required by downstream operations + queue.append(branching_action) + action_names = action_properties_mapping[branching_action]["outputs_from"] # Get the list of actions whose output will be used + if(len(action_names)==1): # if only output of one action is required + key = action_names[0]+"-output" + output = pickle.loads(redis_instace.get(key)) + action_properties_mapping[branching_action]["arguments"] = output + else: + for item in action_names: + key = item+"-output" + output = pickle.loads(redis_instace.get(key)) + output_list.append(output) + action_properties_mapping[branching_action]["arguments"] = output_list + + else: + output_list = [] # List to store the output of actions whose outputs are required by downstream operations + queue.append(alternate_action) + action_names = action_properties_mapping[alternate_action]["outputs_from"] # Get the list of actions whose output will be used + if(len(action_names)==1): # if only output of one action is required + key = action_names[0]+"-output" + output = pickle.loads(redis_instace.get(key)) + action_properties_mapping[alternate_action]["arguments"] = output + else: + for item in action_names: + key = item+"-output" + output = pickle.loads(redis_instace.get(key)) + output_list.append(output) + action_properties_mapping[alternate_action]["arguments"] = output_list + if(condition_op=="greater_than_equals"): - pass + + if(isinstance(action_properties_mapping[action]["condition"]["target"], str)): + target = action_properties_mapping[action]["condition"]["target"] + else: + target=int(action_properties_mapping[action]["condition"]["target"]) + + if(result>=target): + output_list = [] # List to store the output of actions whose outputs are required by downstream operations + queue.append(branching_action) + action_names = action_properties_mapping[branching_action]["outputs_from"] # Get the list of actions whose output will be used + if(len(action_names)==1): # if only output of one action is required + key = action_names[0]+"-output" + output = pickle.loads(redis_instace.get(key)) + action_properties_mapping[branching_action]["arguments"] = output + else: + for item in action_names: + key = item+"-output" + output = pickle.loads(redis_instace.get(key)) + output_list.append(output) + action_properties_mapping[branching_action]["arguments"] = output_list + + else: + output_list = [] # List to store the output of actions whose outputs are required by downstream operations + queue.append(alternate_action) + action_names = action_properties_mapping[alternate_action]["outputs_from"] # Get the list of actions whose output will be used + if(len(action_names)==1): # if only output of one action is required + key = action_names[0]+"-output" + output = pickle.loads(redis_instace.get(key)) + action_properties_mapping[alternate_action]["arguments"] = output + else: + for item in action_names: + key = item+"-output" + output = pickle.loads(redis_instace.get(key)) + output_list.append(output) + action_properties_mapping[alternate_action]["arguments"] = output_list + if(condition_op=="less_than"): - pass + + if(isinstance(action_properties_mapping[action]["condition"]["target"], str)): + target = action_properties_mapping[action]["condition"]["target"] + else: + target=int(action_properties_mapping[action]["condition"]["target"]) + + if(resultFUNC IDS",dag_metadata) - print('\n') - redis_instace.flushdb() - print("Cleaned up in-memory intermediate outputs successfully\n") if(isinstance(reply,list)): res = {"dag_id": dag_metadata["dag_id"], @@ -274,5 +423,5 @@ def execute_dag(dag_name,request): } dag_responses.append(res) + - \ No newline at end of file diff --git a/controlplane/trigger_gateway.py b/controlplane/trigger_gateway.py index a22dcef..6a33fc2 100644 --- a/controlplane/trigger_gateway.py +++ b/controlplane/trigger_gateway.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 import subprocess + import threading -import queue import json import os -import time -from flask import Flask, request,jsonify,send_file +from flask import Flask, request,jsonify import pymongo import orchestrator @@ -15,6 +14,9 @@ import validate_trigger app = Flask(__name__) +lock = threading.Lock() + + action_url_mappings = {} #Store action->url mappings action_properties_mapping = {} #Stores the action name and its corresponding properties responses = [] @@ -200,9 +202,9 @@ def view_dags(): json_data = json.dumps(data, default=str) json_string ='{"dag":'+str(json_data)+'}' data = json.loads(json_string) - # Format the JSON string with indentation - formatted_json = json.dumps(data, indent=4) - return formatted_json + # # Format the JSON string with indentation + # formatted_json = json.dumps(data, indent=4) + return data @app.route('/view/triggers',methods=['GET']) def view_triggers(): @@ -280,59 +282,89 @@ def execute_action(action_name): data = {"status": 404 ,"failure_reason":e} return data +# def write_url_params_to_file(url, params, file_path): +# # Check if the URL already exists in the file +# try: +# with open(file_path, 'r') as file: +# existing_urls = set(':'.join(line.strip().split(':')[:3]) for line in file if line.strip()) +# if url in existing_urls: +# return +# except FileNotFoundError: +# pass + +# # Append the new URL and parameters to the file +# try: +# with open(file_path, 'a') as file: +# file.write(f"{url}:{params}\n") +# except IOError: +# print("An error occurred while writing to the file.") + # EXAMPLE URL: http://10.129.28.219:5001/run/mydagtrigger @app.route('/run/', methods=['GET', 'POST']) def orchestrate_dag(trigger_name): + + # write_url_params_to_file(request.url, request.json, 'requests.txt') + + + orchestrator.dag_responses = [] try: triggers = validate_trigger.get_trigger_json(trigger_name) - if(len(triggers)==0): + if len(triggers) == 0: return {"response": "the given trigger is not registered in DAGit trigger store"} else: thread_list = [] - result_queue = queue.Queue() - if(triggers[0]['type']=='dag'): - dags = triggers[0]['dags'] - arguments = request.json + if triggers[0]['type'] == 'dag': + # dags = triggers[0]['dags'] + try: - for dag in dags: - thread_list.append(threading.Thread(target=orchestrator.execute_dag, args=[dag,arguments])) - for thread in thread_list: - thread.start() - for thread in thread_list: - thread.join() + + # If only 1 dag execute it directly without thread. + + no_of_dags = len(triggers[0]['dags']) + + if no_of_dags==1: + orchestrator.execute_dag(triggers[0]['dags'][0],request.json) + return {"response": orchestrator.dag_responses, "status": 200} - return {"response":orchestrator.dag_responses,"status":200} - + else: + + for dag in triggers[0]['dags']: + thread_list.append(threading.Thread(target=orchestrator.execute_dag, args=[dag, request.json])) + for thread in thread_list: + thread.start() + for thread in thread_list: + thread.join() + + return {"response": orchestrator.dag_responses, "status": 200} + except Exception as e: print(e) - return {"response":"failed","status":400} - + return {"response": "failed", "status": 400} + else: try: functions = triggers[0]['functions'] arguments = request.json + # with lock: for function in functions: - thread_list.append(threading.Thread(target=orchestrator.execute_action, args=[function,arguments])) + thread_list.append(threading.Thread(target=orchestrator.execute_action, args=[function, arguments])) for thread in thread_list: thread.start() for thread in thread_list: thread.join() - return {"response":orchestrator.function_responses,"status":200} + return {"response": orchestrator.function_responses, "status": 200} except Exception as e: print(e) - return {"response":"failed","status":400} - + return {"response": "failed", "status": 400} - except Exception as e: print(e) - data = {"status": 404 ,"message":"failed"} + data = {"status": 404, "message": "failed"} return data if __name__ == '__main__': - app.run(host='0.0.0.0', port=5001) - + app.run(host='0.0.0.0', port=5001) \ No newline at end of file diff --git a/controlplane/trigger_specifications/sentiment_analysis_trigger.json b/controlplane/trigger_specifications/sentiment_analysis_trigger.json new file mode 100644 index 0000000..db16323 --- /dev/null +++ b/controlplane/trigger_specifications/sentiment_analysis_trigger.json @@ -0,0 +1,6 @@ +{ + "trigger_name": "text_sentiment_analysis_trigger", + "type":"dag", + "dags": ["text-sentiment-analysis"], + "functions":"" +} \ No newline at end of file diff --git a/function_modules/calculate_text_sentiment/Dockerfile b/function_modules/calculate_text_sentiment/Dockerfile new file mode 100644 index 0000000..84ac29e --- /dev/null +++ b/function_modules/calculate_text_sentiment/Dockerfile @@ -0,0 +1,11 @@ +FROM openwhisk/python3action + +RUN apk --update add --no-cache g++ lapack-dev gfortran openssl ca-certificates py-openssl jpeg-dev zlib-dev libjpeg + +RUN pip install textblob nltk newspaper3k + +RUN pip install pandas + +RUN python -m textblob.download_corpora + + diff --git a/function_modules/calculate_text_sentiment/build.,txt b/function_modules/calculate_text_sentiment/build.,txt new file mode 100644 index 0000000..e6c5ac8 --- /dev/null +++ b/function_modules/calculate_text_sentiment/build.,txt @@ -0,0 +1,4 @@ +sudo ./buildAndPush.sh 10.129.28.219:5000/text-sentiment-analysis +./register.sh /calculate-sentiment-api /calculate-sentiment-path calculate_sentiment --response-type=json +wsk -i action create calculate_sentiment --docker 10.129.28.219:5000/text-sentiment-analysis sentiment.py --web=true --timeout=420000 -m 2048 +wsk -i action update calculate_sentiment --docker 10.129.28.219:5000/text-sentiment-analysis sentiment.py --timeout=420000 -m 4096 \ No newline at end of file diff --git a/function_modules/calculate_text_sentiment/buildAndPush.sh b/function_modules/calculate_text_sentiment/buildAndPush.sh new file mode 100755 index 0000000..85c75ce --- /dev/null +++ b/function_modules/calculate_text_sentiment/buildAndPush.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# This script will build the docker image and push it to dockerhub. +# +# Usage: buildAndPush.sh imageName +# +# Dockerhub image names look like "username/appname" and must be all lower case. +# For example, "janesmith/calculator" + +IMAGE_NAME=$1 +echo "Using $IMAGE_NAME as the image name" + +# Make the docker image +docker build -t $IMAGE_NAME . +if [ $? -ne 0 ]; then + echo "Docker build failed" + exit +fi +docker push $IMAGE_NAME +if [ $? -ne 0 ]; then + echo "Docker push failed" + exit +fi + diff --git a/function_modules/calculate_text_sentiment/params.json b/function_modules/calculate_text_sentiment/params.json new file mode 100755 index 0000000..7b975bf --- /dev/null +++ b/function_modules/calculate_text_sentiment/params.json @@ -0,0 +1,7 @@ +{ + "processed_data": ["Although mathematics is extensively used for modeling phenomena, the fundamental truths of mathematics are independent from any scientific experimentation", + "This led to split mathematics into pure mathematics and applied mathematics, the latter being often considered as having a lower value among mathematical purists", + "Many of the theories developed for applications were found interesting from the point of view of pure mathematics, and many results of pure mathematics were shown to have applications outside mathematics; in turn, the study of these applications may give new insights on the \"pure theory\"", + "The Mathematics Subject Classification has a section for \"general applied mathematics\" but does not mention \"pure mathematics\"", + "[g] Rigorous reasoning is not specific to mathematics, but, in mathematics, the standard of rigor is much higher than elsewhere"] +} \ No newline at end of file diff --git a/function_modules/calculate_text_sentiment/requirements.txt b/function_modules/calculate_text_sentiment/requirements.txt new file mode 100644 index 0000000..898dd61 --- /dev/null +++ b/function_modules/calculate_text_sentiment/requirements.txt @@ -0,0 +1,5 @@ +tweepy +textblob +pandas +nltk +newspaper3k \ No newline at end of file diff --git a/function_modules/calculate_text_sentiment/sentiment.py b/function_modules/calculate_text_sentiment/sentiment.py new file mode 100644 index 0000000..af71e08 --- /dev/null +++ b/function_modules/calculate_text_sentiment/sentiment.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import os +import json +import sys +from textblob import TextBlob + + +def main(params): + activation_id = os.environ.get('__OW_ACTIVATION_ID') + # params = json.loads(sys.argv[1]) + sentences = params["processed_data"] + sentiments = [] + for sentence in sentences: + blob = TextBlob(sentence) + sentiment = blob.sentiment.polarity + sentiments.append(sentiment) + + + + print(json.dumps({ "activation_id": str(activation_id), + "sentiments" : sentiments + })) + + return({"activation_id": str(activation_id), + "sentiments":sentiments + }) + + + +if __name__ == "__main__": + main(params) diff --git a/function_modules/create_sentiment_report/Dockerfile b/function_modules/create_sentiment_report/Dockerfile new file mode 100644 index 0000000..84ac29e --- /dev/null +++ b/function_modules/create_sentiment_report/Dockerfile @@ -0,0 +1,11 @@ +FROM openwhisk/python3action + +RUN apk --update add --no-cache g++ lapack-dev gfortran openssl ca-certificates py-openssl jpeg-dev zlib-dev libjpeg + +RUN pip install textblob nltk newspaper3k + +RUN pip install pandas + +RUN python -m textblob.download_corpora + + diff --git a/function_modules/create_sentiment_report/build.,txt b/function_modules/create_sentiment_report/build.,txt new file mode 100644 index 0000000..84cabdd --- /dev/null +++ b/function_modules/create_sentiment_report/build.,txt @@ -0,0 +1,4 @@ +sudo ./buildAndPush.sh 10.129.28.219:5000/text-sentiment-analysis +./register.sh /create_sentiment_report-api /create_sentiment_report-path create_sentiment_report --response-type=json +wsk -i action create create_sentiment_report --docker 10.129.28.219:5000/text-sentiment-analysis sentiment_report.py --web=true --timeout=420000 -m 2048 +wsk -i action update create_sentiment_report --docker 10.129.28.219:5000/text-sentiment-analysis sentiment_report.py --timeout=420000 -m 4096 \ No newline at end of file diff --git a/function_modules/create_sentiment_report/buildAndPush.sh b/function_modules/create_sentiment_report/buildAndPush.sh new file mode 100755 index 0000000..85c75ce --- /dev/null +++ b/function_modules/create_sentiment_report/buildAndPush.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# This script will build the docker image and push it to dockerhub. +# +# Usage: buildAndPush.sh imageName +# +# Dockerhub image names look like "username/appname" and must be all lower case. +# For example, "janesmith/calculator" + +IMAGE_NAME=$1 +echo "Using $IMAGE_NAME as the image name" + +# Make the docker image +docker build -t $IMAGE_NAME . +if [ $? -ne 0 ]; then + echo "Docker build failed" + exit +fi +docker push $IMAGE_NAME +if [ $? -ne 0 ]; then + echo "Docker push failed" + exit +fi + diff --git a/function_modules/create_sentiment_report/params.json b/function_modules/create_sentiment_report/params.json new file mode 100755 index 0000000..7b975bf --- /dev/null +++ b/function_modules/create_sentiment_report/params.json @@ -0,0 +1,7 @@ +{ + "processed_data": ["Although mathematics is extensively used for modeling phenomena, the fundamental truths of mathematics are independent from any scientific experimentation", + "This led to split mathematics into pure mathematics and applied mathematics, the latter being often considered as having a lower value among mathematical purists", + "Many of the theories developed for applications were found interesting from the point of view of pure mathematics, and many results of pure mathematics were shown to have applications outside mathematics; in turn, the study of these applications may give new insights on the \"pure theory\"", + "The Mathematics Subject Classification has a section for \"general applied mathematics\" but does not mention \"pure mathematics\"", + "[g] Rigorous reasoning is not specific to mathematics, but, in mathematics, the standard of rigor is much higher than elsewhere"] +} \ No newline at end of file diff --git a/function_modules/create_sentiment_report/sentiment_report.py b/function_modules/create_sentiment_report/sentiment_report.py new file mode 100644 index 0000000..90e8df6 --- /dev/null +++ b/function_modules/create_sentiment_report/sentiment_report.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import os +import json +import sys +import re +import pandas as pd + + + +def main(params): + activation_id = os.environ.get('__OW_ACTIVATION_ID') + + sentences = params["__ow_body"][0]["processed_data"] + + sentiments = params["__ow_body"][1]["sentiments"] + + # Combine sentences and sentiments into a list of dictionaries + data = [{"Sentence": sentence, "Sentiment": sentiment} for sentence, sentiment in zip(sentences, sentiments)] + + # Create a DataFrame from the list of dictionaries + df = pd.DataFrame(data) + + # Convert DataFrame to a formatted string with cleaned formatting + report = df.to_string(index=False) + report = re.sub(r'\n +', '\n', report) + report = report.strip() + + + print(json.dumps({ "activation_id": str(activation_id), + "report" : report + })) + + return({"activation_id": str(activation_id), + "report":report + }) + + + +if __name__ == "__main__": + main(params) diff --git a/function_modules/url_to_summary/Dockerfile b/function_modules/url_to_summary/Dockerfile new file mode 100644 index 0000000..84ac29e --- /dev/null +++ b/function_modules/url_to_summary/Dockerfile @@ -0,0 +1,11 @@ +FROM openwhisk/python3action + +RUN apk --update add --no-cache g++ lapack-dev gfortran openssl ca-certificates py-openssl jpeg-dev zlib-dev libjpeg + +RUN pip install textblob nltk newspaper3k + +RUN pip install pandas + +RUN python -m textblob.download_corpora + + diff --git a/function_modules/url_to_summary/build.,txt b/function_modules/url_to_summary/build.,txt new file mode 100644 index 0000000..e186fa9 --- /dev/null +++ b/function_modules/url_to_summary/build.,txt @@ -0,0 +1,4 @@ +sudo ./buildAndPush.sh 10.129.28.219:5000/text-sentiment-analysis +./register.sh /fetch-sentences-api /fetch-sentences-path fetch_sentences --response-type=json +wsk -i action create fetch_sentences --docker 10.129.28.219:5000/text-sentiment-analysis data_processing.py --web=true --timeout=420000 -m 2098 +wsk -i action update fetch_sentences --docker 10.129.28.219:5000/text-sentiment-analysis data_processing.py --timeout=420000 -m 4096 \ No newline at end of file diff --git a/function_modules/url_to_summary/buildAndPush.sh b/function_modules/url_to_summary/buildAndPush.sh new file mode 100755 index 0000000..85c75ce --- /dev/null +++ b/function_modules/url_to_summary/buildAndPush.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# This script will build the docker image and push it to dockerhub. +# +# Usage: buildAndPush.sh imageName +# +# Dockerhub image names look like "username/appname" and must be all lower case. +# For example, "janesmith/calculator" + +IMAGE_NAME=$1 +echo "Using $IMAGE_NAME as the image name" + +# Make the docker image +docker build -t $IMAGE_NAME . +if [ $? -ne 0 ]; then + echo "Docker build failed" + exit +fi +docker push $IMAGE_NAME +if [ $? -ne 0 ]; then + echo "Docker push failed" + exit +fi + diff --git a/function_modules/url_to_summary/data_processing.py b/function_modules/url_to_summary/data_processing.py new file mode 100644 index 0000000..5ff3abf --- /dev/null +++ b/function_modules/url_to_summary/data_processing.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import os +import json +import sys +from textblob import TextBlob +from newspaper import Article +import re + + +def main(params): + activation_id = os.environ.get('__OW_ACTIVATION_ID') + # params = json.loads(sys.argv[1]) + url = params["url"] + article = Article(url) + article.download() + article.parse() + article.nlp() + data = article.summary + + # Remove newlines and numbers in square brackets + data = re.sub(r'\n', ' ', data) + data = re.sub(r'\[\d+\]', '', data) + + # Split summary into sentences based on periods + sentences = re.split(r'\.', data) + + # Remove leading/trailing whitespaces and empty sentences + sentences = [sentence.strip() for sentence in sentences if sentence.strip()] + + print(json.dumps({ "activation_id": str(activation_id), + "processed_data" : sentences + })) + + return({"activation_id": str(activation_id), + + "processed_data":sentences + }) + + + +if __name__ == "__main__": + main(params) diff --git a/function_modules/url_to_summary/params.json b/function_modules/url_to_summary/params.json new file mode 100755 index 0000000..f1d65c2 --- /dev/null +++ b/function_modules/url_to_summary/params.json @@ -0,0 +1,3 @@ +{ + "url":"https://en.wikipedia.org/wiki/Mathematics" +} \ No newline at end of file diff --git a/function_modules/url_to_summary/requirements.txt b/function_modules/url_to_summary/requirements.txt new file mode 100644 index 0000000..898dd61 --- /dev/null +++ b/function_modules/url_to_summary/requirements.txt @@ -0,0 +1,5 @@ +tweepy +textblob +pandas +nltk +newspaper3k \ No newline at end of file diff --git a/load_test.py b/load_test.py new file mode 100644 index 0000000..d558672 --- /dev/null +++ b/load_test.py @@ -0,0 +1,41 @@ +import requests +import concurrent.futures + +# Define the URL and parameters +url = "http://10.129.28.219:5001/run/text_sentiment_analysis_trigger" +params = { + "url": "https://en.wikipedia.org/wiki/Mathematics" +} + +# Function to send a request +def send_request(url, params): + response = requests.post(url, json=params) + return response.json() + +# Number of parallel requests +num_requests = 500 + +# Create a ThreadPoolExecutor +executor = concurrent.futures.ThreadPoolExecutor() + +# Submit the requests in parallel +futures = [executor.submit(send_request, url, params) for _ in range(num_requests)] + +# Wait for all the requests to complete +concurrent.futures.wait(futures) + +status = [] +# Process the responses +for future in futures: + response = future.result() + # Process the response as needed + status.append(response["status"]) + +print(status) + +success_count = status.count(200) + +if(num_requests == success_count): + print("Success") +else: + print("Failure")