[fa23] hw09 get-help updates (#484)

* always prompt for non-required feedback

* consent & context

* consent confirmation

* shorten consent message, new feedback format

* bump version number

* revert version

client/cli/ok.py

@@ -132,7 +132,9 @@ def parse_input(command_input=None):
     experiment.add_argument('--collab', action='store_true',
                         help="launch collaborative programming environment")
     experiment.add_argument('--get-help', action='store_true',
-                        help="receive 61A-bot feedback on your code")                    
+                        help="receive 61A-bot feedback on your code")   
+    experiment.add_argument('--consent', action='store_true',
+                        help="get 61A-bot research consent")                 
 
     # Debug information
     debug = parser.add_argument_group('ok developer debugging options')

client/protocols/help.py

@@ -14,17 +14,33 @@
 import time
 import sys
 import re
+import os
+import pickle
+import hmac
 
 from client.utils.printer import print_error
 
 class HelpProtocol(models.Protocol):
 
     SERVER = 'https://61a-bot-backend.zamfi.net'
     HELP_ENDPOINT = SERVER + '/get-help-cli'
-    FEEDBACK_PROBABILITY = 0.25
+    FEEDBACK_PROBABILITY = 1
+    FEEDBACK_REQUIRED = False
     FEEDBACK_ENDPOINT = SERVER + '/feedback'
     FEEDBACK_KEY = 'jfv97pd8ogybhilq3;orfuwyhiulae'
+    FEEDBACK_MESSAGE = "The hint was... (Hit Enter to skip)\n1) Helpful, all fixed\n2) Helpful, not all fixed\n3) Not helpful, but made sense\n4) Not helpful, didn't make sense\n5) Misleading/Wrong\n"
+    FEEDBACK_OPTIONS = set([str(i) for i in range(1, 6)])
+    HELP_TYPE_MESSAGE = "\nThe hint could have included...\n1) More debugging\n2) An example\n3) Template code\n4) Conceptual refresher\n5) More info\n"
+    HELP_TYPE_OPTIONS = set([str(i) for i in range(1, 6)])
+    HELP_OPTIONS = {"y", "yes"}
     HELP_KEY = 'jfv97pd8ogybhilq3;orfuwyhiulae'
+    AG_PREFIX = "————————————————————————\nThe following is an automated report from an autograding tool that may indicate a failed test case or a syntax error. Consider it in your response.\n\n"
+    GET_CONSENT = True
+    CONSENT_CACHE = '.ok_consent'
+    NO_CONSENT_OPTIONS = {"n", "no", "0", "-1", }
+    CONSENT_MESSAGE = "Can we collect your de-identified data for research directed by Prof. Narges Norouzi (EECS faculty member unaffiliated with this course)? Your consent is voluntary and does not affect your ability to use this tool or your course grade. For more information visit https://cs61a.org/articles/61a-bot\n\nYou can change your response at any time by running `python3 ok --consent`."
+    CONTEXT_CACHE = '.ok_context'
+    CONTEXT_LENGTH = 3
 
     def run(self, messages):
         config = config_utils._get_config(self.args.config)
@@ -41,25 +57,31 @@ def run(self, messages):
                 active_function = name
                 break
 
-        autograder_output = messages.get('autograder_output', '')
         get_help = self.args.get_help
         help_payload = None
 
         if (failed or get_help) and (config.get('src', [''])[0][:2] == 'hw'):
-            res = input("Would you like to receive 61A-bot feedback on your code (y/N)? ")
+            res = input("Would you like to receive 61A-bot feedback on your code (y/N)? ").lower()
             print()
-            if res == "y":
+            if res in self.HELP_OPTIONS:
                 filename = config['src'][0]
                 code = open(filename, 'r').read()
+                autograder_output = messages.get('autograder_output', '')
+                email = messages.get('email') or '<unknown from CLI>'
+                consent = self._get_consent(email)
+                context = self._get_context(email)
+                curr_message = {'role': 'user', 'content': code}
                 help_payload = {
-                    'email': messages.get('email') or '<unknown from CLI>',
+                    'email': email,
                     'promptLabel': 'Get_help',
                     'hwId': re.findall(r'hw(\d+)\.(py|scm|sql)', filename)[0][0],
                     'activeFunction': active_function,
-                    'code': code,
-                    'codeError': autograder_output,
+                    'code': code if len(context) == 0 else '',
+                    'codeError': self.AG_PREFIX + autograder_output,
                     'version': 'v2',
-                    'key': self.HELP_KEY
+                    'key': self.HELP_KEY,
+                    'consent': consent,
+                    'messages': context + [curr_message]
                 }
 
         if help_payload:
@@ -79,29 +101,132 @@ def animate():
             except Exception as e:
                 print_error("Error generating hint. Please try again later.")
                 return
-            print(help_response.get('output', "An error occurred. Please try again later."))
+            if 'output' not in help_response:
+                print_error("An error occurred. Please try again later.")
+                return
+
+            hint = help_response.get('output')
+            print(hint)
             print()
 
+            self._append_context(email, curr_message)
+            self._append_context(email, {'role': 'assistant', 'content': hint})
+
             random.seed(int(time.time()))
             if random.random() < self.FEEDBACK_PROBABILITY:
-                print("Please indicate whether the feedback you received was helpful or not.")
-                print("1) It was helpful.")
-                print("-1) It was not helpful.")
-                feedback = None
-                while feedback not in {"1", "-1"}:
-                    if feedback is None:
-                        feedback = input("? ")
+                time.sleep(1)
+                self._get_feedback(help_response.get('requestId'))
+
+    def _get_feedback(self, req_id):
+        print(self.FEEDBACK_MESSAGE)
+        feedback = input("? ")
+        if feedback in self.FEEDBACK_OPTIONS:
+            if feedback == "3":
+                print(self.HELP_TYPE_MESSAGE)
+                help_type = None
+                while help_type not in self.HELP_TYPE_OPTIONS:
+                    if help_type is None:
+                        help_type = input("? ")
                     else:
-                        feedback = input("-- Please select a provided option. --\n? ")
-                print("\nThank you for your feedback.\n")
-                req_id = help_response.get('requestId')
-                if req_id:
-                    feedback_payload = {
-                        'version': 'v2',
-                        'key': self.FEEDBACK_KEY,
-                        'requestId': req_id,
-                        'feedback': feedback
-                    }
-                    feedback_response = requests.post(self.FEEDBACK_ENDPOINT, json=feedback_payload).json() 
+                        help_type = input("-- Please select a provided option. --\n? ")
+
+                feedback += ',' + help_type
+
+            print("\nThank you for your feedback.\n")
+
+            if req_id:
+                feedback_payload = {
+                    'version': 'v2',
+                    'key': self.FEEDBACK_KEY,
+                    'requestId': req_id,
+                    'feedback': feedback
+                }
+                feedback_response = requests.post(self.FEEDBACK_ENDPOINT, json=feedback_payload).json()
+                return feedback_response.get('status')
+
+    def _get_binary_feedback(self, req_id):
+        skip_str = ' Hit Enter to skip.' if not self.FEEDBACK_REQUIRED else ''
+        print(f"Please indicate whether the feedback you received was helpful or not.{skip_str}")
+        print("1) It was helpful.")
+        print("-1) It was not helpful.")
+        feedback = None
+        if self.FEEDBACK_REQUIRED:
+            while feedback not in {"1", "-1"}:
+                if feedback is None:
+                    feedback = input("? ")
+                else:
+                    feedback = input("-- Please select a provided option. --\n? ")
+        else:
+            feedback = input("? ")
+            if feedback not in {"1", "-1"}:
+                print()
+                return
+        print("\nThank you for your feedback.\n")
+
+        if req_id:
+            feedback_payload = {
+                'version': 'v2',
+                'key': self.FEEDBACK_KEY,
+                'requestId': req_id,
+                'feedback': feedback
+            }
+            feedback_response = requests.post(self.FEEDBACK_ENDPOINT, json=feedback_payload).json()
+            return feedback_response.get('status')
+
+    def _mac(self, key, value):
+        mac = hmac.new(key.encode('utf-8'), digestmod='sha512')
+        mac.update(repr(value).encode('utf-8'))
+        return mac.hexdigest()
+
+    def _get_consent(self, email):
+        if self.GET_CONSENT:
+            if self.CONSENT_CACHE in os.listdir() and not self.args.consent:
+                try:
+                    with open(self.CONSENT_CACHE, 'rb') as f:
+                        data = pickle.load(f)
+                        if not hmac.compare_digest(data.get('mac'), self._mac(email, data.get('consent'))):
+                            os.remove(self.CONSENT_CACHE)
+                            return self._get_consent(email)
+                    return data.get('consent')
+                except:
+                    os.remove(self.CONSENT_CACHE)
+                    return self._get_consent(email)
+            else:
+                print(self.CONSENT_MESSAGE)
+                res = input("\n(Y/n)? ").lower()
+                consent = res not in self.NO_CONSENT_OPTIONS
+                if consent:
+                    print("\nYou have consented.\n")
+                else:
+                    print("\nYou have not consented.\n")
+                with open(self.CONSENT_CACHE, 'wb') as f:
+                    pickle.dump({'consent': consent, 'mac': self._mac(email, consent)}, f, protocol=pickle.HIGHEST_PROTOCOL)
+                return consent
+        else:
+            return False
+
+    def _get_context(self, email, full=False):
+        if self.CONTEXT_CACHE in os.listdir():
+            try:
+                with open(self.CONTEXT_CACHE, 'rb') as f:
+                    data = pickle.load(f)
+                    if not hmac.compare_digest(data.get('mac'), self._mac(email, data.get('context', []))):
+                        os.remove(self.CONTEXT_CACHE)
+                        return self._get_context(email)
+                if full:
+                    return data.get('context', [])
+                else:
+                    return data.get('context', [])[-(self.CONTEXT_LENGTH * 2):]
+            except:
+                os.remove(self.CONTEXT_CACHE)
+                return self._get_context(email)
+        else:
+            return []
+
+    def _append_context(self, email, message):
+        context = self._get_context(email, full=True)
+        context.append(message)
+        with open(self.CONTEXT_CACHE, 'wb') as f:
+            pickle.dump({'context': context, 'mac': self._mac(email, context)}, f, protocol=pickle.HIGHEST_PROTOCOL)
 
 protocol = HelpProtocol