Merge branch 'development'

.gitignore

@@ -8,6 +8,7 @@
 *.py[co]
 *.sw[ponm]
 *~
+.DS_Store
 .certificates/
 .coverage/
 .idea/

CHANGELOG.md

@@ -2,7 +2,38 @@
 <!-- Use the tags Added, Changed, Deprecated, Removed, Fixed, Security, and
      Contributor to describe changes -->
 
-## [0.8.2]
+## [0.8.4]
+###### 2018-07-30
+
+### Added
+* Workflows now support environment variables. These are top-level
+  arguments to a workflow. These are then exposed on the execution page
+  allowing users to modify the most important variables in a workflow
+  without modifying the workflow itself.
+* Added a health check endpoint at the /heath endpoint
+
+### Changed
+* The Metrics page now defaults to showing workflow metrics instead of
+  app metrics
+
+
+### Fixed
+* Action results and workflow results stream now filter for the
+  currently-executing workflow. This eliminates many issues experienced
+  by multiple users executing workflows concurrently from the workflow
+  editor
+* Fixed an error which caused the Scheduler to not execute workflows
+* Fixed another bug in the scheduler in which the scheduled workflows
+  would not persist across server restarts
+* A bug where messages couldn't be sent
+* A bug where modifying more than one device at a time on the playbook
+  editor would cause the workflow to be invalidated
+* Some database configuration bugs when used with non-SQLite databases
+* Fixed a bug which wouldn't allow a user to a abort a workflow if it
+  was pending execution.
+
+
+## [0.8.3]
 ###### 2018-06-14
 
 ### Added

requirements.txt

@@ -22,3 +22,5 @@ marshmallow >= 2.15, < 3.0.0
 marshmallow-sqlalchemy >= 0.13.0
 pynacl
 alembic
+healthcheck
+psycopg2-binary

tests/__init__.py

@@ -28,10 +28,12 @@
            'test_disk_subscription',
            'test_event_dispatcher',
            'test_events',
+           'test_environment_variable',
            'test_transform',
            'test_condition',
            'test_condition_transform_validation',
            'test_filtered_sse_stream',
+           'test_health_endpoint',
            'test_helper_functions',
            'test_input_validation',
            'test_interface_event_dispatch_helpers',

tests/suites.py

@@ -19,7 +19,7 @@ def add_tests_to_suite(suite, test_modules):
                   test_redis_cache_adapter, test_redis_subscription, test_disk_subscription, test_sse_stream,
                   test_filtered_sse_stream, test_notification_stream, test_workflow_status, test_problem,
                   test_workflow_results_stream, test_streamable_blueprint, test_console_stream, test_disk_pubsub_cache,
-                  test_make_cache]
+                  test_make_cache, test_health_endpoint]
 server_suite = TestSuite()
 add_tests_to_suite(server_suite, __server_tests)
 
@@ -35,7 +35,7 @@ def add_tests_to_suite(suite, test_modules):
 execution_suite = TestSuite()
 add_tests_to_suite(execution_suite, __execution_tests)
 
-__workflow_tests = [test_simple_workflow, test_workflow_manipulation]
+__workflow_tests = [test_simple_workflow, test_workflow_manipulation, test_environment_variable]
 workflow_suite = TestSuite()
 add_tests_to_suite(workflow_suite, __workflow_tests)
 

tests/testWorkflows/environmentVariables.playbook

@@ -0,0 +1,36 @@
+{
+    "name": "environmentVariables",
+    "workflows": [
+        {
+            "actions": [
+                {
+                    "action_name": "repeatBackToMe",
+                    "app_name": "HelloWorldBounded",
+                    "arguments": [
+                        {
+                            "name": "call",
+                            "reference": "d524843a-9caa-4fc1-aa0e-e8dfbdba7490"
+                        }
+                    ],
+                    "device_id": {
+                        "name": "__device__",
+                        "value": 1
+                    },
+                    "id": "4269a5b5-bbb7-49d9-becc-31152ccc29b7",
+                    "name": "start"
+                }
+            ],
+            "environment_variables": [
+                {
+                    "id": "d524843a-9caa-4fc1-aa0e-e8dfbdba7490",
+                    "name": "call",
+                    "value": "CHANGE INPUT",
+                    "type": "str"
+                }
+            ],
+            "branches": [],
+            "name": "environmentVariables",
+            "start": "4269a5b5-bbb7-49d9-becc-31152ccc29b7"
+        }
+    ]
+}

tests/test_case_server.py

@@ -15,6 +15,7 @@
 from walkoff.server.endpoints.cases import convert_subscriptions, split_subscriptions
 from walkoff.server.returncodes import *
 from walkoff.serverdb.casesubscription import CaseSubscription
+from walkoff.case.database import Case, Event
 
 
 class TestCaseServer(ServerTestCase):
@@ -34,8 +35,13 @@ def setUp(self):
         current_app.running_context.case_logger = self.logger
 
     def tearDown(self):
+        current_app.running_context.case_db.session.rollback()
         for case in current_app.running_context.case_db.session.query(case_database.Case).all():
             current_app.running_context.case_db.session.delete(case)
+        for event in current_app.running_context.case_db.session.query(Event).all():
+            current_app.running_context.case_db.session.delete(event)
+        for link in current_app.running_context.case_db.session.query(case_database._CaseEventLink):
+            current_app.running_context.case_db.session.delete(link)
         current_app.running_context.case_db.commit()
         for case in CaseSubscription.query.all():
             db.session.delete(case)
@@ -439,3 +445,42 @@ def test_send_cases_to_workers(self):
                 current_app.running_context.case_db.session.commit()
             send_all_cases_to_workers()
             mock_update.assert_has_calls(expected)
+
+    def test_cases_pagination(self):
+        for i in range(40):
+            self.create_case(str(i))
+
+        response = self.get_with_status_check('/api/cases', headers=self.headers)
+        self.assertEqual(len(response), 20)
+
+        response = self.get_with_status_check('/api/cases?page=2', headers=self.headers)
+        self.assertEqual(len(response), 20)
+
+        response = self.get_with_status_check('/api/cases?page=3', headers=self.headers)
+        self.assertEqual(len(response), 0)
+
+    def test_read_event(self):
+        case = Case(name='test_case')
+        event = Event(note='test_note')
+        current_app.running_context.case_db.session.add(case)
+        current_app.running_context.case_db.session.commit()
+        current_app.running_context.case_db.add_event(event, [case.id])
+
+        response = self.get_with_status_check('/api/events/{}'.format(event.id), headers=self.headers)
+        self.assertEqual(response['id'], event.id)
+        self.assertEqual(response['note'], event.note)
+
+    def test_update_event_note(self):
+        case = Case(name='test_case')
+        event = Event(note='test_note')
+        current_app.running_context.case_db.session.add(case)
+        current_app.running_context.case_db.session.commit()
+        current_app.running_context.case_db.add_event(event, [case.id])
+
+        data = {'id': event.id, 'note': 'CHANGE NOTE'}
+        self.put_with_status_check('/api/events', headers=self.headers, data=json.dumps(data),
+                                   content_type='application/json', status_code=SUCCESS)
+
+        response = self.get_with_status_check('/api/events/{}'.format(event.id), headers=self.headers)
+        self.assertEqual(response['id'], event.id)
+        self.assertEqual(response['note'], 'CHANGE NOTE')

tests/test_console_stream.py

@@ -1,5 +1,6 @@
 import json
 from copy import copy
+from uuid import uuid4
 
 from flask import Response
 from mock import patch
@@ -33,18 +34,39 @@ def test_console_log_callback(self, mock_publish):
         data = {'app_name': 'App1', 'action_name': 'action1', 'level': 'WARN', 'message': 'some_message'}
         console_log_callback(sender, data=data)
         expected = format_console_data(sender, data=data)
-        mock_publish.assert_called_once_with(expected, event='log')
+        mock_publish.assert_called_once_with(expected, event='log', subchannels=sender['execution_id'])
 
-    @patch.object(console_stream, 'stream')
-    def test_stream_endpoint(self, mock_stream):
-        mock_stream.return_value = Response('something', status=SUCCESS)
+    def call_stream(self, execution_id=None):
         post = self.test_client.post('/api/auth', content_type="application/json",
                                      data=json.dumps(dict(username='admin', password='admin')), follow_redirects=True)
         key = json.loads(post.get_data(as_text=True))['access_token']
-        response = self.test_client.get('/api/streams/console/log?access_token={}'.format(key))
-        mock_stream.assert_called_once_with()
+        url = '/api/streams/console/log?access_token={}'.format(key)
+        if execution_id:
+            url += '&workflow_execution_id={}'.format(execution_id)
+        return self.test_client.get(url)
+
+    @patch.object(console_stream, 'stream')
+    def test_stream_endpoint(self, mock_stream):
+        mock_stream.return_value = Response('something', status=SUCCESS)
+        execution_id = str(uuid4())
+        response = self.call_stream(execution_id=execution_id)
+        mock_stream.assert_called_once_with(subchannel=execution_id)
         self.assertEqual(response.status_code, SUCCESS)
 
+    @patch.object(console_stream, 'stream')
+    def test_stream_endpoint_invalid_uuid(self, mock_stream):
+        mock_stream.return_value = Response('something', status=SUCCESS)
+        response = self.call_stream(execution_id='invalid')
+        mock_stream.assert_not_called()
+        self.assertEqual(response.status_code, BAD_REQUEST)
+
+    @patch.object(console_stream, 'stream')
+    def test_stream_endpoint_no_execution_id(self, mock_stream):
+        mock_stream.return_value = Response('something', status=SUCCESS)
+        response = self.call_stream()
+        mock_stream.assert_not_called()
+        self.assertEqual(response.status_code, BAD_REQUEST)
+
     @patch.object(console_stream, 'stream')
     def check_stream_endpoint_no_key(self, mock_stream):
         mock_stream.return_value = Response('something', status=SUCCESS)

tests/test_device_server.py

@@ -38,7 +38,7 @@ def test_read_all_devices(self):
         app = App(name=self.test_app_name, devices=[device1, device2])
         self.app.running_context.execution_db.session.add(app)
         self.app.running_context.execution_db.session.commit()
-        response = self.get_with_status_check('/api/devices', headers=self.headers, status_code=SUCCESS)
+        response = self.get_with_status_check('/api/devices?page=1', headers=self.headers, status_code=SUCCESS)
         expected_device1 = device1.as_json()
         expected_device1['app_name'] = 'TestApp'
         expected_device2 = device2.as_json()
@@ -278,3 +278,30 @@ def test_import_apps_devices(self):
             # Checks if test field is a subset of the device json fields
             self.assertTrue(any(x for x in device['fields'] if set(x) not in set(
                 {k.encode("utf-8"): str(v).encode("utf-8") for k, v in field.items()})))
+
+    def test_device_pagination(self):
+        fields_json = [{'name': 'test_name', 'type': 'integer', 'encrypted': False},
+                       {'name': 'test2', 'type': 'string', 'encrypted': False}]
+        walkoff.config.app_apis = {self.test_app_name: {'devices': {'test_type': {'fields': fields_json}}}}
+        app = App(name=self.test_app_name)
+        self.app.running_context.execution_db.session.add(app)
+        self.app.running_context.execution_db.session.commit()
+
+        for i in range(40):
+            device_json = {'app_name': 'TestApp', 'name': str(i), 'type': 'test_type',
+                           'fields': [{'name': 'test_name', 'value': 123}, {'name': 'test2', 'value': 'something'}]}
+            self.post_with_status_check('/api/devices', headers=self.headers, data=json.dumps(device_json),
+                                        status_code=OBJECT_CREATED, content_type='application/json')
+
+        response = self.get_with_status_check('/api/devices?page=1', headers=self.headers, status_code=SUCCESS)
+        self.assertEqual(len(response), 20)
+        devices = [str(i) for i in range(20)]
+        for device in response:
+            self.assertIn(device['name'], devices)
+        response = self.get_with_status_check('/api/devices?page=2', headers=self.headers, status_code=SUCCESS)
+        self.assertEqual(len(response), 20)
+        devices = [str(i) for i in range(20, 40)]
+        for device in response:
+            self.assertIn(device['name'], devices)
+        response = self.get_with_status_check('/api/devices?page=3', headers=self.headers, status_code=SUCCESS)
+        self.assertEqual(len(response), 0)

tests/test_environment_variable.py

@@ -0,0 +1,27 @@
+import unittest
+
+import walkoff.appgateway
+from tests.util import execution_db_help
+from tests.util import initialize_test_config
+from walkoff.executiondb.environment_variable import EnvironmentVariable
+
+
+class TestAction(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        initialize_test_config()
+        execution_db_help.setup_dbs()
+
+    @classmethod
+    def tearDownClass(cls):
+        walkoff.appgateway.clear_cache()
+        execution_db_help.tear_down_execution_db()
+
+    def __compare_init(self, elem, name, value, description):
+        self.assertEqual(elem.name, name)
+        self.assertEqual(elem.value, value)
+        self.assertEqual(elem.description, description)
+
+    def test_init_default(self):
+        env_var = EnvironmentVariable(name='test_name', value='test_value', description='test_description')
+        self.__compare_init(env_var, 'test_name', 'test_value', 'test_description')

tests/test_health_endpoint.py

@@ -0,0 +1,16 @@
+from tests.util.servertestcase import ServerTestCase
+
+
+class TestHealthEndpoint(ServerTestCase):
+
+    def test_endpoint(self):
+        response = self.get_with_status_check('/health', status_code=200)
+        expected_checks = {
+            'check_cache',
+            'check_server_db',
+            'check_execution_db'
+        }
+        self.assertEqual(response['status'], 'success')
+        checks = {result['checker'] for result in response['results']}
+        self.assertSetEqual(checks, expected_checks)
+        self.assertTrue(all(result['passed'] for result in response['results']))

tests/test_messaging_endpoints.py

@@ -355,3 +355,15 @@ def test_get_all_notifications_more_than_maximum_some_unread(self):
         notifications = self.get_notifications(self.user1)
         self.assertEqual(len(notifications), max_notifications)
         self.assertTrue(all(not notification['is_read'] for notification in notifications))
+
+    def test_message_pagination(self):
+        for i in range(38):
+            TestMessagingEndpoints.make_message([self.user1.user])
+        response = self.get_with_status_check('/api/messages', headers=self.user1.header, status_code=SUCCESS)
+        self.assertEqual(len(response), 20)
+        response = self.get_with_status_check('/api/messages?page=2', headers=self.user1.header,
+                                              status_code=SUCCESS)
+        self.assertEqual(len(response), 20)
+        response = self.get_with_status_check('/api/messages?page=3', headers=self.user1.header,
+                                              status_code=SUCCESS)
+        self.assertEqual(len(response), 0)

tests/test_redis_cache_adapter.py

@@ -2,6 +2,7 @@
 
 from tests.util.mock_objects import MockRedisCacheAdapter
 from walkoff.cache import unsubscribe_message
+import mock
 
 
 class TestRedisCacheAdapter(TestCase):

tests/test_scheduledtasks_database.py

@@ -5,18 +5,13 @@
 
 from tests.util import initialize_test_config
 from tests.util.execution_db_help import setup_dbs
+from tests.util.servertestcase import ServerTestCase
 from walkoff.scheduler import InvalidTriggerArgs
 from walkoff.serverdb import db
 from walkoff.serverdb.scheduledtasks import ScheduledTask
 
 
-class TestScheduledTask(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        initialize_test_config()
-        cls.context = current_app.test_request_context()
-        cls.context.push()
-        setup_dbs()
+class TestScheduledTask(ServerTestCase):
 
     def setUp(self):
         self.date_trigger = {'type': 'date', 'args': {'run_date': '2017-01-25 10:00:00'}}

tests/test_scheduledtasks_server.py

@@ -230,3 +230,19 @@ def test_stop_from_stopped(self):
 
     def test_stop_does_not_exist(self):
         self.take_action(404, 'stop', status_code=OBJECT_DNE_ERROR)
+
+    def test_scheduler_pagination(self):
+        workflow_id = [str(uuid4())]
+        for i in range(40):
+            data = {"name": str(i), "workflows": workflow_id, "task_trigger": self.date_scheduler}
+            self.post_with_status_check('/api/scheduledtasks', data=json.dumps(data), headers=self.headers,
+                                        content_type='application/json', status_code=OBJECT_CREATED)
+
+        response = self.get_with_status_check('/api/scheduledtasks', headers=self.headers)
+        self.assertEqual(len(response), 20)
+
+        response = self.get_with_status_check('/api/scheduledtasks?page=2', headers=self.headers)
+        self.assertEqual(len(response), 20)
+
+        response = self.get_with_status_check('/api/scheduledtasks?page=3', headers=self.headers)
+        self.assertEqual(len(response), 0)