diff --git a/README.md b/README.md index 9b85bf3fb559ccf9cc7c602789b2109475083a06..c0ea33a493476b243e32cf346026a2a1022c9545 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ On submission to the GradeIt turn-in form, a student's code is loaded into the s ### Grade -At grade time, each student's code is again loaded into the solution by move.py. The emulator is wiped of all previous installations and screenshots. The application is compiled and installed on the target device and integration tests are then run. Finally, `tests.py` is run which reads JUnit test result XML files and assigns points according to `criteria.xml`. After grading is complete the grading commands along with any screenshots and APK files produced are copied out of the remote execution environment. +At grade time, each student's code is again loaded into the solution by move.py. The emulator is wiped of all previous installations and screenshots. The application is compiled and installed on the target device and integration tests are then run. Finally, `tests.py` is run which reads JUnit test result XML files and assigns points according to GitGrade rubric. After grading is complete the grading commands along with any screenshots and APK files produced are copied out of the remote execution environment. -Due to the nature of integration testing, **only one grading may be executed at a time**. ~~For GradeIt, this means "simultaneous requests" on the grading runner **must** be set to 1.~~ This is not necessarily true anymore as emulator access has a mutex lock, but there is a timeout. +Due to the nature of integration testing, **only one grading may be executed at a time**. ~~For GitGrade, this means "simultaneous requests" on the grading runner **must** be set to 1.~~ This is not necessarily true anymore as emulator access has a mutex lock, but there is a timeout. ## Support Files @@ -122,4 +122,7 @@ Since GitGrade stores all assignment information in a database, the only necessa ~~We added the `testRegex` attribute. The order of the ``s does not matter unless the `testSubtracts` attribute is specified on an item. If this attribute is specified, test cases matching the item regex will be excluded from consideration in subsequent items.~~ -This file is rendered obsolete now that we're using GitGrade. The entire scoresheet as stored in GitGrade's database is available to the scripts at runtime via the `GITGRADE_ASSIGNMENT_CRITERIA` environment variable. It is in the same JSON export format as that seen on the scoresheet export page of GitGrade. Jeremy added the ability to add custom attributes to scoresheet items, so we can just add the `testRegex`s there. For this change, one would most likely have to modify `grade.sh` and `scripts/tests.py`. \ No newline at end of file +This file is rendered obsolete now that we're using GitGrade. The entire scoresheet as stored in GitGrade's database is available to the scripts at runtime via the `GITGRADE_ASSIGNMENT_CRITERIA` environment variable. It is in the same JSON export format as that seen on the scoresheet export page of GitGrade. Jeremy added the ability to add custom attributes to scoresheet items, so we can just add the `testRegex`s there. For this change, one would most likely have to modify `grade.sh` and `scripts/tests.py`. + +### GitGrade Assignment Rubric +Using the Item Specific Tags feature in the rubric (designated by the tag symbol button next to an item), `testRegex` may be added as a key along with the value of a regex for the tests to apply the score automatically. diff --git a/grade.sh b/grade.sh index 663248007bbff8b97e189861128c17540a089a7d..8d27f301dc6801ce5740529e422d6a71033861e4 100755 --- a/grade.sh +++ b/grade.sh @@ -120,6 +120,7 @@ ssh $SSH_ARGS $SSH_HOST /bin/bash << EOSSH # Award points based on test output. export GITGRADE_STUDENT_TURNIN_TIMESTAMP="${GITGRADE_STUDENT_TURNIN_TIMESTAMP}" export GITGRADE_ASSIGNMENT_DUE="${GITGRADE_ASSIGNMENT_DUE}" + export GITGRADE_STUDENT_CRITERIA="${GITGRADE_STUDENT_CRITERIA}" scripts/tests.py >&3 if [ ! -z "\${ADB_HOST}" ]; then scripts/screenshots.py diff --git a/scripts/tests.py b/scripts/tests.py index e6adf7998352574f022b2fb83557ae4396e7daac..568e98da2b90a06539a7ca6bc67c5552eac70121 100644 --- a/scripts/tests.py +++ b/scripts/tests.py @@ -4,32 +4,47 @@ # the assignment's critera.xml. # # Written by Ryan Rowe. +# Modified for GitGrade tags by Jeremy Zhang # Copyright 2019 UW CSE. import os import re +import json from math import ceil from junitparser import JUnitXml from glob import glob -from xml.etree import ElementTree from dateutil import parser from datetime import timedelta def get_items_from_criteria(criteria): # Recursively find items and associated test regexes. - categories = [([], cat) for cat in criteria.findall("category")] + categories = [] items = [] remainder = [] + + for item in criteria: + if item.get("type") == "category": + categories.append(([], item)) + elif item.get("type") == "item": + tags = item.get("tags", {}) + if tags.get("testRegex", None): + items.append([], item) + while categories: path, category = categories.pop() - path.append(category.attrib["name"]) + path.append(category["name"]) # Handle recursive categories. - categories.extend((path.copy(), cat) for cat in category.findall("category")) - - items.extend((path, item) for item in category.findall("item") if "testRegex" in item.attrib) - remainder.extend((path, item) for item in category.findall("item") if "testRegex" not in item.attrib) + for item in category["items"]: + if item.get("type") == "category": + categories.append((path.copy(), item)) + elif item.get("type") == "item": + tags = item.get("tags", {}) + if tags.get("testRegex", None): + items.append((path, item)) + else: + remainder.append((path, item)) return items, remainder @@ -41,13 +56,13 @@ def assign_junit_points(items): tests.update({"{}.{}".format(test.classname, test.name): test.result is None for test in results}) # Assign points to each item based on test results. - with open("grader_output.html", "a+") as output: + with open("grader_output.html", "a+", encoding='utf-8') as output: output.write("") output.write("

Test Results

\n") for path, item in items: - regex = re.compile(item.attrib["testRegex"]) - pts = float(item.attrib["max"]) - name = item.text + regex = re.compile(item["tags"]["testRegex"]) + pts = float(item["maxPoints"]) + name = item["description"] required_tests = list(filter(regex.search, tests)) @@ -71,7 +86,7 @@ def assign_junit_points(items): output.write("    {}
".format(short_test)) print("point_set {}".format(pass_rate * pts)) - if "testSubtracts" in item.attrib: + if "testSubtracts" in item.get("tags", {}): for test in required_tests: tests.pop(test) @@ -90,7 +105,7 @@ def get_assignment_due_date(): def assign_late_penalty(remainder): # Determine late days (if late days supported by criteria). - item = list(((path, item) for path, item in remainder if "late" in item.text.lower())) + item = list(((path, item) for path, item in remainder if "late" in item["description"].lower())) if len(item): # There is an item which describes late days. path, item = item[0] @@ -105,9 +120,8 @@ def assign_late_penalty(remainder): def main(): # Load assignment grading criteria. - with open("criteria.xml", "r") as f: - criteria = ElementTree.parse(f).getroot() - items, remainder = get_items_from_criteria(criteria) + criteria = json.loads(os.environ['GITGRADE_STUDENT_CRITERIA']) + items, remainder = get_items_from_criteria(criteria["criteria"]) assign_junit_points(items)