diff --git a/evaluation_function/main.py b/evaluation_function/main.py index 2e170cb..ae972f4 100644 --- a/evaluation_function/main.py +++ b/evaluation_function/main.py @@ -105,16 +105,16 @@ def main(): - If 2+ args provided: File-based communication (last 2 args are input/output paths) - Otherwise: RPC/IPC server mode using lf_toolkit """ - # # Check for file-based communication - # # shimmy passes input and output file paths as the last two arguments - # if len(sys.argv) >= 3: - # input_path = sys.argv[-2] - # output_path = sys.argv[-1] + # Check for file-based communication + # shimmy passes input and output file paths as the last two arguments + if len(sys.argv) >= 3: + input_path = sys.argv[-2] + output_path = sys.argv[-1] - # # Verify they look like file paths (basic check) - # if not input_path.startswith('-') and not output_path.startswith('-'): - # handle_file_based_communication(input_path, output_path) - # return + # Verify they look like file paths (basic check) + if not input_path.startswith('-') and not output_path.startswith('-'): + handle_file_based_communication(input_path, output_path) + return # Fall back to RPC/IPC server mode server = create_server() diff --git a/evaluation_function/test/test_correction.py b/evaluation_function/test/test_correction.py index 88c3c4d..5bb519c 100644 --- a/evaluation_function/test/test_correction.py +++ b/evaluation_function/test/test_correction.py @@ -207,5 +207,121 @@ def test_non_minimal_fsa_fails_when_required(self, equivalent_dfa): assert any(e.code == ErrorCode.NOT_MINIMAL for e in result.fsa_feedback.errors) +# ============================================================================= +# Test Epsilon Transitions (End-to-End) +# ============================================================================= + +class TestEpsilonTransitionCorrection: + """Test the full correction pipeline with ε-NFA inputs.""" + + def test_epsilon_nfa_vs_equivalent_dfa_correct(self): + """ε-NFA student answer equivalent to DFA expected should be correct.""" + # ε-NFA accepts exactly "a": q0 --ε--> q1 --a--> q2 + student_enfa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + # DFA accepts exactly "a": s0 --a--> s1 + expected_dfa = make_fsa( + states=["s0", "s1"], + alphabet=["a"], + transitions=[ + {"from_state": "s0", "to_state": "s1", "symbol": "a"}, + ], + initial="s0", + accept=["s1"], + ) + result = analyze_fsa_correction(student_enfa, expected_dfa) + assert isinstance(result, Result) + assert result.is_correct is True + + def test_epsilon_nfa_vs_different_dfa_incorrect(self): + """ε-NFA accepting 'a' vs DFA accepting 'b' should be incorrect.""" + student_enfa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a", "b"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + expected_dfa = make_fsa( + states=["s0", "s1"], + alphabet=["a", "b"], + transitions=[ + {"from_state": "s0", "to_state": "s1", "symbol": "b"}, + ], + initial="s0", + accept=["s1"], + ) + result = analyze_fsa_correction(student_enfa, expected_dfa) + assert isinstance(result, Result) + assert result.is_correct is False + assert result.fsa_feedback is not None + assert len(result.fsa_feedback.errors) > 0 + + def test_multi_epsilon_nfa_vs_dfa_correct(self): + """ε-NFA for (a|b) with branching epsilons should match equivalent DFA.""" + student_enfa = make_fsa( + states=["q0", "q1", "q2", "q3"], + alphabet=["a", "b"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q0", "to_state": "q2", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q3", "symbol": "a"}, + {"from_state": "q2", "to_state": "q3", "symbol": "b"}, + ], + initial="q0", + accept=["q3"], + ) + expected_dfa = make_fsa( + states=["s0", "s1"], + alphabet=["a", "b"], + transitions=[ + {"from_state": "s0", "to_state": "s1", "symbol": "a"}, + {"from_state": "s0", "to_state": "s1", "symbol": "b"}, + ], + initial="s0", + accept=["s1"], + ) + result = analyze_fsa_correction(student_enfa, expected_dfa) + assert isinstance(result, Result) + assert result.is_correct is True + + def test_epsilon_nfa_structural_info_reports_nondeterministic(self): + """ε-NFA should have structural info reporting non-deterministic.""" + student_enfa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + expected_dfa = make_fsa( + states=["s0", "s1"], + alphabet=["a"], + transitions=[ + {"from_state": "s0", "to_state": "s1", "symbol": "a"}, + ], + initial="s0", + accept=["s1"], + ) + result = analyze_fsa_correction(student_enfa, expected_dfa) + assert result.fsa_feedback is not None + assert result.fsa_feedback.structural is not None + assert result.fsa_feedback.structural.is_deterministic is False + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/evaluation_function/test/test_validation.py b/evaluation_function/test/test_validation.py index 3cf711a..9423774 100644 --- a/evaluation_function/test/test_validation.py +++ b/evaluation_function/test/test_validation.py @@ -314,55 +314,185 @@ def test_isomorphic_dfas(self): initial="s0", accept=["s1"], ) - assert are_isomorphic(fsa_user, fsa_sol).ok - - -# ============================================================================= -# Test Minimality -# ============================================================================= - -@pytest.fixture -def dfa_accepts_a(): - """DFA that accepts exactly 'a'.""" - return make_fsa( - states=["q0", "q1", "q2"], - alphabet=["a", "b"], - transitions=[ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q0", "to_state": "q2", "symbol": "b"}, - {"from_state": "q1", "to_state": "q2", "symbol": "a"}, - {"from_state": "q1", "to_state": "q2", "symbol": "b"}, - {"from_state": "q2", "to_state": "q2", "symbol": "a"}, - {"from_state": "q2", "to_state": "q2", "symbol": "b"}, - ], - initial="q0", - accept=["q1"] - ) - - -class TestCheckMinimality: - """Test check_minimality function.""" - - def test_minimal_dfa(self, dfa_accepts_a): - result = is_minimal(dfa_accepts_a) - assert result.ok - assert result.value is True - - def test_non_minimal_dfa_with_unreachable(self): - non_minimal = make_fsa( - states=["q0", "q1", "q2", "unreachable"], + assert are_isomorphic(fsa_user, fsa_sol) == [] + + +class TestEpsilonTransitions: + """Tests for epsilon transition handling across the validation pipeline.""" + + def test_valid_fsa_with_epsilon_unicode(self): + """ε-NFA with Unicode ε should pass structural validation.""" + fsa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + assert is_valid_fsa(fsa) == [] + + def test_valid_fsa_with_epsilon_string(self): + """ε-NFA with 'epsilon' string should pass structural validation.""" + fsa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "epsilon"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + assert is_valid_fsa(fsa) == [] + + def test_valid_fsa_with_empty_string_epsilon(self): + """ε-NFA with empty string epsilon should pass structural validation.""" + fsa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": ""}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + assert is_valid_fsa(fsa) == [] + + def test_epsilon_nfa_is_not_deterministic(self): + """ε-NFA should be flagged as non-deterministic.""" + fsa = make_fsa( + states=["q0", "q1"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + ], + initial="q0", + accept=["q1"], + ) + errors = is_deterministic(fsa) + assert len(errors) > 0 + assert ErrorCode.NOT_DETERMINISTIC in [e.code for e in errors] + + def test_accepts_string_via_epsilon_closure(self): + """ε-NFA should accept 'a' by following q0 --ε--> q1 --a--> q2.""" + fsa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + assert accepts_string(fsa, "a") == [] + + def test_rejects_string_with_epsilon_nfa(self): + """ε-NFA that accepts 'a' should reject empty string.""" + fsa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + errors = accepts_string(fsa, "") + assert len(errors) > 0 + + def test_accepts_empty_string_via_epsilon(self): + """ε-NFA should accept empty string when initial reaches accept via ε.""" + fsa = make_fsa( + states=["q0", "q1"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + ], + initial="q0", + accept=["q1"], + ) + assert accepts_string(fsa, "") == [] + + def test_epsilon_nfa_equivalent_to_dfa(self): + """ε-NFA and DFA accepting the same language should be equivalent.""" + enfa = make_fsa( + states=["q0", "q1", "q2"], + alphabet=["a"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q2", "symbol": "a"}, + ], + initial="q0", + accept=["q2"], + ) + dfa = make_fsa( + states=["s0", "s1"], + alphabet=["a"], + transitions=[ + {"from_state": "s0", "to_state": "s1", "symbol": "a"}, + ], + initial="s0", + accept=["s1"], + ) + assert fsas_accept_same_language(enfa, dfa) == [] + + def test_epsilon_nfa_not_equivalent_to_different_dfa(self): + """ε-NFA and DFA accepting different languages should not be equivalent.""" + enfa = make_fsa( + states=["q0", "q1", "q2"], alphabet=["a", "b"], transitions=[ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q0", "to_state": "q2", "symbol": "b"}, + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, {"from_state": "q1", "to_state": "q2", "symbol": "a"}, - {"from_state": "q1", "to_state": "q2", "symbol": "b"}, - {"from_state": "q2", "to_state": "q2", "symbol": "a"}, - {"from_state": "q2", "to_state": "q2", "symbol": "b"}, - {"from_state": "unreachable", "to_state": "unreachable", "symbol": "a"}, ], initial="q0", - accept=["q1"] + accept=["q2"], ) - result = is_minimal(non_minimal) - assert not result.ok + dfa = make_fsa( + states=["s0", "s1"], + alphabet=["a", "b"], + transitions=[ + {"from_state": "s0", "to_state": "s1", "symbol": "b"}, + ], + initial="s0", + accept=["s1"], + ) + errors = fsas_accept_same_language(enfa, dfa) + assert len(errors) > 0 + + def test_multi_epsilon_nfa_equivalent_to_dfa(self): + """ε-NFA for (a|b) with branching epsilons should match equivalent DFA.""" + # q0 --ε--> q1, q0 --ε--> q2, q1 --a--> q3, q2 --b--> q3 + enfa = make_fsa( + states=["q0", "q1", "q2", "q3"], + alphabet=["a", "b"], + transitions=[ + {"from_state": "q0", "to_state": "q1", "symbol": "ε"}, + {"from_state": "q0", "to_state": "q2", "symbol": "ε"}, + {"from_state": "q1", "to_state": "q3", "symbol": "a"}, + {"from_state": "q2", "to_state": "q3", "symbol": "b"}, + ], + initial="q0", + accept=["q3"], + ) + dfa = make_fsa( + states=["s0", "s1"], + alphabet=["a", "b"], + transitions=[ + {"from_state": "s0", "to_state": "s1", "symbol": "a"}, + {"from_state": "s0", "to_state": "s1", "symbol": "b"}, + ], + initial="s0", + accept=["s1"], + ) + assert fsas_accept_same_language(enfa, dfa) == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/evaluation_function/validation/validation.py b/evaluation_function/validation/validation.py index 7cc3bbe..787835c 100644 --- a/evaluation_function/validation/validation.py +++ b/evaluation_function/validation/validation.py @@ -3,6 +3,8 @@ from evaluation_function.schemas.result import StructuralInfo from ..algorithms.minimization import hopcroft_minimization +from ..algorithms.nfa_to_dfa import nfa_to_dfa, is_deterministic as is_dfa_check +from ..algorithms.epsilon_closure import epsilon_closure_set, build_epsilon_transition_map from ..schemas import FSA, ValidationError, ErrorCode, ElementHighlight, ValidationResult @@ -92,7 +94,7 @@ def is_valid_fsa(fsa: FSA) -> ValidationResult[bool]: ) ) ) - if t.symbol not in alphabet: + if t.symbol not in alphabet and t.symbol not in ("ε", "epsilon", ""): errors.append( ValidationError( message=f"Symbol '{t.symbol}' not in alphabet.", @@ -125,6 +127,31 @@ def is_deterministic(fsa: FSA) -> ValidationResult[bool]: errors: List[ValidationError] = [] seen = set() + + # First check if FSA is structurally valid + structural_errors = is_valid_fsa(fsa) + if structural_errors: + return structural_errors + + # Check for epsilon transitions (makes FSA non-deterministic) + for t in fsa.transitions: + if t.symbol in ("ε", "epsilon", ""): + errors.append( + ValidationError( + message=f"Your FSA has an epsilon (ε) transition from '{t.from_state}' to '{t.to_state}'. A DFA cannot have epsilon transitions.", + code=ErrorCode.NOT_DETERMINISTIC, + severity="error", + highlight=ElementHighlight( + type="transition", + from_state=t.from_state, + to_state=t.to_state, + symbol=t.symbol + ), + suggestion="Remove epsilon transitions to make this a DFA, or note that your FSA is an NFA/ε-NFA, which is also valid!" + ) + ) + if errors: + return errors for t in fsa.transitions: key = (t.from_state, t.symbol) @@ -144,6 +171,8 @@ def is_deterministic(fsa: FSA) -> ValidationResult[bool]: ) seen.add(key) + return errors + return ( ValidationResult.success(True) if not errors @@ -287,7 +316,21 @@ def accepts_string(fsa: FSA, string: str) -> ValidationResult[bool]: if not valid.ok: return valid - current_states: Set[str] = {fsa.initial_state} +def accepts_string(fsa: FSA, string: str) -> List[ValidationError]: + """ + Simulate the FSA on a string, with full ε-transition support. + Returns [] if accepted, else a ValidationError. + """ + # First check if FSA is structurally valid + structural_errors = is_valid_fsa(fsa) + if structural_errors: + return structural_errors + + # Build epsilon transition map for ε-closure computation + epsilon_trans = build_epsilon_transition_map(fsa.transitions) + + # Start with ε-closure of the initial state + current_states: Set[str] = epsilon_closure_set({fsa.initial_state}, epsilon_trans) for symbol in string: if symbol not in fsa.alphabet: @@ -297,19 +340,21 @@ def accepts_string(fsa: FSA, string: str) -> ValidationResult[bool]: code=ErrorCode.INVALID_SYMBOL, severity="error" ) - ]) + ] - next_states = { - t.to_state - for s in current_states - for t in fsa.transitions - if t.from_state == s and t.symbol == symbol - } + next_states = set() + for state in current_states: + for t in fsa.transitions: + if t.from_state == state and t.symbol == symbol: + next_states.add(t.to_state) - if not next_states: - return ValidationResult.failure(False, [ + # Compute ε-closure of the states reached after reading the symbol + current_states = epsilon_closure_set(next_states, epsilon_trans) + + if not current_states: + return [ ValidationError( - message=f"String '{string}' rejected.", + message=f"String '{string}' rejected: no transition on symbol '{symbol}'", code=ErrorCode.TEST_CASE_FAILED, severity="error" ) @@ -346,7 +391,57 @@ def fsas_accept_same_string(fsa1: FSA, fsa2: FSA, string: str) -> ValidationResu code=ErrorCode.LANGUAGE_MISMATCH, severity="error" ) - ]) + ] + return [] + + +def fsas_accept_same_language(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: + # Convert NFA/ε-NFA to DFA before minimization (Hopcroft requires DFA input) + if not is_dfa_check(fsa1): + fsa1 = nfa_to_dfa(fsa1) + if not is_dfa_check(fsa2): + fsa2 = nfa_to_dfa(fsa2) + + fsa1_min = hopcroft_minimization(fsa1) + fsa2_min = hopcroft_minimization(fsa2) + return are_isomorphic(fsa1_min, fsa2_min) + # """ + # Approximate check for language equivalence by testing all strings up to max_length. + # Returns [] if equivalent, else a ValidationError. + # """ + # errors = [] + + # if set(fsa1.alphabet) != set(fsa2.alphabet): + # errors.append( + # ValidationError( + # message=f"Alphabets of FSAs differ: FSA1 alphabet = {set(fsa1.alphabet)}, FSA2 alphabet = {set(fsa2.alphabet)}", + # code=ErrorCode.LANGUAGE_MISMATCH, + # severity="error" + # ) + # ) + # return errors + + # alphabet = fsa1.alphabet + + # # Check empty string + # empty_string_error = fsas_accept_same_string(fsa1, fsa2, "") + # if empty_string_error: + # return empty_string_error + + # for length in range(1, max_length + 1): + # for s in product(alphabet, repeat=length): + # string = ''.join(s) + # err = fsas_accept_same_string(fsa1, fsa2, string) + # if err: + # errors.append( + # ValidationError( + # message=f"FSAs differ on string '{string}' of length {length}", + # code=ErrorCode.LANGUAGE_MISMATCH, + # severity="error" + # ) + # ) + # return errors # stop at first counterexample + # return errors return ValidationResult.success(True)