Merge pull request #2019 from cvisionai/rc/1.4.3

1.4.3 staging area

Author: bctcvai

.circleci/config.yml

@@ -186,15 +186,48 @@ jobs:
     - attach_workspace:
         at: ~/
     - run:
-        name: Initialize REST tests
-        command: ssh lightsail 'cd tator && make testinit'
+        name: Install xmlstarlet
+        command: sudo apt-get update && sudo apt-get install -y xmlstarlet
     - run:
         name: Run REST tests
         command: |
-          mkdir -p rest-test-results;
-          ssh lightsail 'cd tator && for i in $(seq 1 3); do make rq-empty && make test && break || sleep 10; done;';
-          ssh lightsail 'test -e tator/rest-junit.xml' && scp lightsail:tator/rest-junit.xml rest-test-results/ || exit 0;
-          grep -q -E '\b[1-9][0-9]*\s+failed\b' rest-test-results/rest-junit.xml && exit 1 || exit 0;
+          # Disable exit-on-error to allow all commands to run
+          set +e
+
+          # Create directory for test results
+          mkdir -p rest-test-results
+
+          # Run test initialization
+          ssh -o "StrictHostKeyChecking=no" -o "ConnectTimeout=10" lightsail 'cd tator && make testinit'
+
+          # Run tests and capture exit status
+          ssh -o "StrictHostKeyChecking=no" -o "ConnectTimeout=10" lightsail 'cd tator && make rq-empty && make test'
+          TEST_EXIT_STATUS=$?
+
+          # Always attempt to copy the XML file
+          ssh -o "StrictHostKeyChecking=no" -o "ConnectTimeout=10" lightsail 'test -e tator/rest-junit.xml' && \
+            scp lightsail:tator/rest-junit.xml rest-test-results/ || echo "No test results found"
+
+          # Check if XML file was copied
+          if [ -f rest-test-results/rest-junit.xml ]; then
+            # Parse XML for failures or errors
+            FAILURES=$(xmlstarlet sel -t -v "//testsuite/@failures" rest-test-results/rest-junit.xml)
+            ERRORS=$(xmlstarlet sel -t -v "//testsuite/@errors" rest-test-results/rest-junit.xml)
+
+            if [ "$FAILURES" -gt 0 ] || [ "$ERRORS" -gt 0 ]; then
+              echo "Test failures or errors detected"
+              exit 1
+            else
+              echo "All tests passed"
+              exit 0
+            fi
+          else
+            echo "No test results found"
+            if [ $TEST_EXIT_STATUS -ne 0 ]; then
+              echo "Tests failed to run"
+            fi
+            exit 1
+          fi
     - store_test_results:
         path: rest-test-results
   front-end-tests:
@@ -209,20 +242,38 @@ jobs:
           rsync -a --ignore-existing /home/circleci/.ssh/ lightsail:/home/ubuntu/.ssh/;
           ssh lightsail 'sed -i "s/circleci/ubuntu/g" ~/.ssh/config';
     - run:
-        name: Install sshfs
-        command: sudo -E apt-get update && sudo -E apt-get -yq --no-install-suggests --no-install-recommends install sshfs
+        name: Install sshfs and xmlstarlet
+        command: sudo -E apt-get update && sudo -E apt-get -yq --no-install-suggests --no-install-recommends install sshfs xmlstarlet
     - run:
         name: Front end tests
         no_output_timeout: 30m
         command: |
-          ssh lightsail 'cd tator && make rq-empty';
-          mkdir -p /tmp/videos;
-          mkdir -p frontend-test-results;
-          ssh lightsail 'mkdir -p /tmp/videos';
-          sshfs -o default_permissions lightsail:/tmp/videos /tmp/videos;
-          ssh lightsail 'mkdir -p /tmp/videos && export PATH=$PATH:$HOME/.local/bin:/snap/bin && export PYTHONUNBUFFERED=1 && cd tator && for i in $(seq 1 1); do if [ -d "test-results" ]; then rm -f test-results/*; else mkdir -p test-results; fi; python3 scripts/test_support/pytest_parallel.py --num-workers 3 test -n 2 --keep --base-url=http://localhost:8080 --browser=chromium --username=admin --password=admin --videos=/tmp/videos -s --junitxml=test-results/frontend-junit.xml && break || sleep 10; done;';
-          ssh lightsail 'test -e tator/test-results/frontend-junit.xml' && scp lightsail:tator/test-results/frontend-junit.xml frontend-test-results/ || exit 0;
-          grep -q -E '\b[1-9][0-9]*\s+failed\b' frontend-test-results/frontend-junit.xml && exit 1 || exit 0;
+          set +e  # Disable exit on error to ensure all commands run
+          ssh lightsail 'cd tator && make rq-empty'
+          mkdir -p /tmp/videos
+          mkdir -p frontend-test-results
+          ssh lightsail 'mkdir -p /tmp/videos'
+          sshfs -o default_permissions lightsail:/tmp/videos /tmp/videos
+          ssh lightsail 'export PATH=$PATH:$HOME/.local/bin:/snap/bin && export PYTHONUNBUFFERED=1 && cd tator && if [ -d "test-results" ]; then rm -f test-results/*; else mkdir -p test-results; fi; python3 scripts/test_support/pytest_parallel.py --num-workers 3 test -n 2 --keep --base-url=http://localhost:8080 --browser=chromium --username=admin --password=admin --videos=/tmp/videos -s --junitxml=test-results/frontend-junit.xml'
+          TEST_EXIT_STATUS=$?
+          ssh lightsail 'test -e tator/test-results/frontend-junit.xml' && scp lightsail:tator/test-results/frontend-junit.xml frontend-test-results/ || echo "No test results found"
+          if [ -f frontend-test-results/frontend-junit.xml ]; then
+            FAILURES=$(xmlstarlet sel -t -v "//testsuite/@failures" frontend-test-results/frontend-junit.xml)
+            ERRORS=$(xmlstarlet sel -t -v "//testsuite/@errors" frontend-test-results/frontend-junit.xml)
+            if [ "$FAILURES" -gt 0 ] || [ "$ERRORS" -gt 0 ]; then
+              echo "Test failures or errors detected"
+              exit 1
+            else
+              echo "All tests passed"
+              exit 0
+            fi
+          else
+            echo "No test results found"
+            if [ $TEST_EXIT_STATUS -ne 0 ]; then
+              echo "Tests failed to run"
+            fi
+            exit 1
+          fi
     - store_test_results:
         path: frontend-test-results
     - store_artifacts:
@@ -235,22 +286,43 @@ jobs:
     steps:
     - attach_workspace:
         at: ~/
+    - run:
+        name: Install xmlstarlet
+        command: sudo apt-get update && sudo apt-get install -y xmlstarlet
     - run:
         name: Copy test directories
         command: |
           ssh lightsail 'rm -rf ~/tator/tatorpy_test && cp -r ~/tator/scripts/packages/tator-py/test ~/tator/tatorpy_test && cp -r ~/tator/scripts/packages/tator-py/examples ~/tator/.';
     - run:
         name: Get API token
         command: |
-          ssh lightsail "curl -d '{"'"username"'": "'"admin"'", "'"password"'": "'"admin"'", "'"refresh"'": true}' -H 'Content-Type: application/json' http://localhost:8080/rest/Token | jq '.token' | xargs printf '%b\n' | tee ~/token.txt;"
+          ssh lightsail "curl -d '{\"username\": \"admin\", \"password\": \"admin\", \"refresh\": true}' -H 'Content-Type: application/json' http://localhost:8080/rest/Token | jq '.token' | xargs printf '%b\n' | tee ~/token.txt;"
           scp lightsail:~/token.txt ~/token.txt
     - run:
         name: Run tator-py tests
         command: |
-          mkdir -p tatorpy-test-results;
-          ssh lightsail 'export TATOR_TOKEN='"'$(cat ~/token.txt)'"'; export TATOR_HOST=http://localhost:8080; export PATH=$PATH:$HOME/.local/bin:/snap/bin && cd tator && for i in $(seq 1 3); do if [ -d "test-results" ]; then rm -f test-results/*; else mkdir -p test-results; fi; pytest tatorpy_test --run-alt-bucket --ignore tatorpy_test/test_algorithm_launch.py --ignore tatorpy_test/test_job_cancel.py --ignore tatorpy_test/test_hosted_template.py -s --keep --junitxml=test-results/tatorpy-junit.xml && break || sleep 10; done;';
-          ssh lightsail 'test -e tator/test-results/tatorpy-junit.xml' && scp lightsail:tator/test-results/tatorpy-junit.xml tatorpy-test-results/ || exit 0;
-          grep -q -E '\b[1-9][0-9]*\s+failed\b' tatorpy-test-results/tatorpy-junit.xml && exit 1 || exit 0;
+          set +e  # Disable exit on error to ensure all commands run
+          mkdir -p tatorpy-test-results
+          ssh lightsail 'export TATOR_TOKEN='"'$(cat ~/token.txt)'"'; export TATOR_HOST=http://localhost:8080; export PATH=$PATH:$HOME/.local/bin:/snap/bin && cd tator && if [ -d "test-results" ]; then rm -f test-results/*; else mkdir -p test-results; fi; pytest tatorpy_test --run-alt-bucket --ignore tatorpy_test/test_algorithm_launch.py --ignore tatorpy_test/test_job_cancel.py --ignore tatorpy_test/test_hosted_template.py -s --keep --junitxml=test-results/tatorpy-junit.xml'
+          TEST_EXIT_STATUS=$?
+          ssh lightsail 'test -e tator/test-results/tatorpy-junit.xml' && scp lightsail:tator/test-results/tatorpy-junit.xml tatorpy-test-results/ || echo "No test results found"
+          if [ -f tatorpy-test-results/tatorpy-junit.xml ]; then
+            FAILURES=$(xmlstarlet sel -t -v "//testsuite/@failures" tatorpy-test-results/tatorpy-junit.xml)
+            ERRORS=$(xmlstarlet sel -t -v "//testsuite/@errors" tatorpy-test-results/tatorpy-junit.xml)
+            if [ "$FAILURES" -gt 0 ] || [ "$ERRORS" -gt 0 ]; then
+              echo "Test failures or errors detected"
+              exit 1
+            else
+              echo "All tests passed"
+              exit 0
+            fi
+          else
+            echo "No test results found"
+            if [ $TEST_EXIT_STATUS -ne 0 ]; then
+              echo "Tests failed to run"
+            fi
+            exit 1
+          fi
     - store_test_results:
         path: tatorpy-test-results
     - run:

api/main/rest/_annotation_query.py

@@ -33,12 +33,14 @@
 
 def _do_object_search(qs, params):
     if params.get("object_search"):
-        qs = get_attribute_psql_queryset_from_query_obj(qs, params.get("object_search"))
+        qs = get_attribute_psql_queryset_from_query_obj(
+            params["project"], qs, params.get("object_search")
+        )
 
     # Used by GET queries
     if params.get("encoded_search"):
         search_obj = json.loads(base64.b64decode(params.get("encoded_search").encode()).decode())
-        qs = get_attribute_psql_queryset_from_query_obj(qs, search_obj)
+        qs = get_attribute_psql_queryset_from_query_obj(params["project"], qs, search_obj)
 
     return qs
 
@@ -141,7 +143,7 @@ def _get_annotation_psql_queryset(project, filter_ops, params, annotation_type):
     )
     if filter_type is not None:
         qs = get_attribute_psql_queryset(
-            ANNOTATION_TYPE_LOOKUP[annotation_type].objects.get(pk=filter_type),
+            ANNOTATION_TYPE_LOOKUP[annotation_type].objects.filter(pk=filter_type).values("pk", "attribute_types").first(),
             qs,
             params,
             filter_ops,
@@ -155,10 +157,10 @@ def _get_annotation_psql_queryset(project, filter_ops, params, annotation_type):
         )
     elif filter_ops or params.get("float_array", None):
         queries = []
-        for entity_type in ANNOTATION_TYPE_LOOKUP[annotation_type].objects.filter(project=project):
+        for entity_type in ANNOTATION_TYPE_LOOKUP[annotation_type].objects.filter(project=project).values('pk', 'attribute_types'):
             sub_qs = get_attribute_psql_queryset(entity_type, qs, params, filter_ops)
             if type(sub_qs) != type(None):
-                queries.append(sub_qs.filter(type=entity_type))
+                queries.append(sub_qs.filter(type=entity_type['pk']))
         logger.info(f"Joining {len(queries)} queries together.")
         sub_qs = queries.pop()
         if queries:
@@ -183,7 +185,9 @@ def _get_annotation_psql_queryset(project, filter_ops, params, annotation_type):
                 media_qs = media_qs.filter(pk__in=section.media.values("id"))
             elif section.dtype == "saved_search":
                 if object_search:
-                    media_qs = get_attribute_psql_queryset_from_query_obj(media_qs, object_search)
+                    media_qs = get_attribute_psql_queryset_from_query_obj(
+                        project, media_qs, object_search
+                    )
                 elif related_object_search:
                     media_state_types = StateType.objects.filter(project=project)
                     media_localization_types = Localization.objects.filter(project=project)
@@ -210,7 +214,7 @@ def _get_annotation_psql_queryset(project, filter_ops, params, annotation_type):
         faux_params = {key.replace("related_", ""): params[key] for key in matches}
         logger.info(faux_params)
         related_matches = []
-        for entity_type in related_media_types:
+        for entity_type in related_media_types.values('pk', 'attribute_types'):
             faux_filter_ops = get_attribute_filter_ops(faux_params, entity_type)
             if faux_filter_ops:
                 related_matches.append(
@@ -236,7 +240,7 @@ def _get_annotation_psql_queryset(project, filter_ops, params, annotation_type):
         related_matches = []
         for entity_type in related_media_types:
             media_qs = Media.objects.filter(project=project, type=entity_type)
-            media_qs = get_attribute_psql_queryset_from_query_obj(media_qs, search_obj)
+            media_qs = get_attribute_psql_queryset_from_query_obj(project, media_qs, search_obj)
             if media_qs.exists():
                 related_matches.append(media_qs)
         if related_matches:
@@ -294,7 +298,7 @@ def get_annotation_queryset(project, params, annotation_type):
         types = ANNOTATION_TYPE_LOOKUP[annotation_type].objects.filter(pk=filter_type)
     else:
         types = ANNOTATION_TYPE_LOOKUP[annotation_type].objects.filter(project=project)
-    for entity_type in types:
+    for entity_type in types.values("pk", "attribute_types"):
         filter_ops.extend(get_attribute_filter_ops(params, entity_type))
     qs = _get_annotation_psql_queryset(project, filter_ops, params, annotation_type)
     return qs