Update tests to new logic

Author: GJFR

bughog/search_strategy/bgb_search.py

@@ -27,12 +27,12 @@ def __init__(self, state_factory: StateFactory, considered_states: Optional[list
         """
         super().__init__(state_factory, 0, considered_states=considered_states)
 
-    def next(self) -> State:
+    def next(self, wait=True) -> State:
         """
         Returns the next state to evaluate.
         """
         # Fetch all evaluated states
-        self._fetch_evaluated_states()
+        self._fetch_evaluated_states(wait=wait)
 
         if self._limit and self._limit <= len(self._considered_states):
             raise SequenceFinished()

bughog/search_strategy/bgb_sequence.py

@@ -25,13 +25,13 @@ def __init__(self, state_factory: StateFactory, limit: int, considered_states: O
         self._unavailability_gap_pairs: set[tuple[State, State]] = set()
         """Tuples in this list are **strict** boundaries of ranges without any available binaries."""
 
-    def next(self) -> State:
+    def next(self, wait=True) -> State:
         """
         Returns the next state to evaluate.
         """
         # Fetch all evaluated states on the first call
         if not self._considered_states:
-            self._fetch_evaluated_states()
+            self._fetch_evaluated_states(wait=wait)
 
         if self._limit and self._limit <= len(self._considered_states):
             raise SequenceFinished()

bughog/search_strategy/composite_search.py

@@ -11,17 +11,17 @@ def __init__(self, state_factory: StateFactory, sequence_limit: int) -> None:
         self.sequence_strategy = BiggestGapBisectionSequence(state_factory, limit=sequence_limit)
         self.search_strategy: Optional[BiggestGapBisectionSearch] = None
 
-    def next(self) -> State:
+    def next(self, wait=True) -> State:
         """
         Returns the next state, based on a sequence strategy and search strategy.
         First, the sequence strategy decides which state to return until it returns the SequenceFinished exception.
         From then on, the search strategy decides which state to return.
         """
         if self.search_strategy is None:
             try:
-                return self.sequence_strategy.next()
+                return self.sequence_strategy.next(wait=wait)
             except SequenceFinished:
                 self.search_strategy = BiggestGapBisectionSearch.create_from_bgb_sequence(self.sequence_strategy)
-                return self.search_strategy.next()
+                return self.search_strategy.next(wait=wait)
         else:
-            return self.search_strategy.next()
+            return self.search_strategy.next(wait=wait)

bughog/search_strategy/sequence_strategy.py

@@ -115,7 +115,7 @@ def __get_closest_available_state(self, target: State, boundaries: tuple[State,
         Return the closest state with an available binary.
         """
         try:
-            states = target.get_previous_and_next_state_with_binary()
+            states = target.get_previous_and_next_state_with_executable()
             states = [state for state in states if state is not None]
             ordered_states = sorted(states, key=lambda x: abs(target.commit_nb - x.commit_nb))
 

bughog/version_control/state/base.py

@@ -40,12 +40,7 @@ def has_same_outcome(self, other: State) -> bool:
         if self.result_variables is None or other.result_variables is None:
             return False
         else:
-            return (
-                ExperimentResult.poc_is_reproduced(self.result_variables) ==
-                ExperimentResult.poc_is_reproduced(other.result_variables) and
-                ExperimentResult.poc_is_dirty(self.result_variables) ==
-                ExperimentResult.poc_is_dirty(other.result_variables)
-            )
+            return ExperimentResult.poc_is_reproduced(self.result_variables) == ExperimentResult.poc_is_reproduced(other.result_variables) and ExperimentResult.poc_is_dirty(self.result_variables) == ExperimentResult.poc_is_dirty(other.result_variables)
 
     @property
     def name(self) -> str:
@@ -131,7 +126,7 @@ def get_executable_source_urls(self) -> list[str]:
         """
         pass
 
-    def get_previous_and_next_state_with_binary(self) -> tuple[State, State]:
+    def get_previous_and_next_state_with_executable(self) -> tuple[State, State]:
         raise NotImplementedError(f'This function is not implemented for {self}')
 
     def __repr__(self) -> str:

test/availability/__init__.py

@@ -3,24 +3,25 @@
 
 import requests
 
-from bughog.evaluation.collectors.collector import Collector, Type
+from bughog.evaluation.collectors.collector import Collector
+from bughog.evaluation.collectors.requests import RequestCollector
 
 
 class TestCollector(unittest.TestCase):
     @staticmethod
     def test_start_stop():
-        collector = Collector([Type.REQUESTS])
-        results = collector.collect_result_result_variables()
-        assert results['requests'] == []
-        assert results['req_vars'] == []
+        collector = Collector([RequestCollector()])
+        raw_results, variables = collector.collect_results()
+        assert raw_results['requests'] == []
+        assert variables == set()
 
         collector.start()
         time.sleep(2)
         collector.stop()
 
-        results = collector.collect_result_result_variables()
-        assert results['requests'] == []
-        assert results['req_vars'] == []
+        raw_results, variables = collector.collect_results()
+        assert raw_results['requests'] == []
+        assert variables == set()
         time.sleep(1)
         # Port should be freed
 
@@ -30,11 +31,12 @@ def test_start_stop():
 
     @staticmethod
     def test_requests():
-        collector = Collector([Type.REQUESTS])
+        collector = Collector([RequestCollector()])
         collector.start()
-        response_data = {'url': 'bughog_testvar=123', 'method': 'GET', 'headers': [], 'content': 'test'}
+        response_data = {'url': 'https://leak.test/report/?bughog_testvar=123', 'method': 'GET', 'headers': [], 'content': 'test'}
         requests.post('http://localhost:5001', json=response_data)
         time.sleep(1)
         collector.stop()
-        results = collector.collect_result_result_variables()
-        assert results['requests'] == [response_data]
+        raw_results, variables = collector.collect_results()
+        assert raw_results['requests'] == [response_data]
+        assert len(variables) == 1 and ('testvar', '123') in variables

test/availability/test_folders.py

@@ -12,30 +12,30 @@ def test_sbg_search_always_available_search(self):
             helper.always_has_binary,
             outcome_func=lambda x: True if x < 50 else False)
         sequence = BiggestGapBisectionSearch(state_factory)
-        index_sequence = [sequence.next().index for _ in range(8)]
+        index_sequence = [sequence.next(wait=False).index for _ in range(8)]
         assert index_sequence == [0, 99, 49, 74, 61, 55, 52, 50]
-        self.assertRaises(SequenceFinished, sequence.next)
+        self.assertRaises(SequenceFinished, lambda: sequence.next(wait=False))
 
     def test_sbg_search_even_available_search(self):
         state_factory = helper.create_state_factory(
             helper.only_has_binaries_for_even,
             outcome_func=lambda x: True if x < 35 else False)
         sequence = BiggestGapBisectionSearch(state_factory)
 
-        assert sequence.next().index == 0
-        assert [state.index for state in sequence._completed_states] == [0]
+        assert sequence.next(wait=False).index == 0
+        assert [state.index for state in sequence._considered_states] == [0]
         assert sequence._unavailability_gap_pairs == set()
 
         while True:
             try:
-                sequence.next()
+                sequence.next(wait=False)
             except SequenceFinished:
                 break
 
-        assert ([state.index for state in sequence._completed_states]
+        assert ([state.index for state in sequence._considered_states]
                 == [0, 24, 30, 32, 34, 36, 48, 98])
 
-        self.assertRaises(SequenceFinished, sequence.next)
+        self.assertRaises(SequenceFinished, lambda: sequence.next(wait=False))
         assert {(first.index, last.index) for (first, last) in sequence._unavailability_gap_pairs} == {(34, 36)}
 
 
@@ -45,23 +45,23 @@ def test_sbg_search_few_available_search(self):
             outcome_func=lambda x: True if x < 35 else False)
         sequence = BiggestGapBisectionSearch(state_factory)
 
-        assert sequence.next().index == 0
-        assert [state.index for state in sequence._completed_states] == [0]
+        assert sequence.next(wait=False).index == 0
+        assert [state.index for state in sequence._considered_states] == [0]
         assert sequence._unavailability_gap_pairs == set()
 
-        assert sequence.next().index == 99
-        assert [state.index for state in sequence._completed_states] == [0, 99]
+        assert sequence.next(wait=False).index == 99
+        assert [state.index for state in sequence._considered_states] == [0, 99]
 
-        assert sequence.next().index == 44
-        assert [state.index for state in sequence._completed_states] == [0, 44, 99]
+        assert sequence.next(wait=False).index == 44
+        assert [state.index for state in sequence._considered_states] == [0, 44, 99]
 
-        assert sequence.next().index == 22
-        assert [state.index for state in sequence._completed_states] == [0, 22, 44, 99]
+        assert sequence.next(wait=False).index == 22
+        assert [state.index for state in sequence._considered_states] == [0, 22, 44, 99]
 
-        assert sequence.next().index == 33
-        assert [state.index for state in sequence._completed_states] == [0, 22, 33, 44, 99]
+        assert sequence.next(wait=False).index == 33
+        assert [state.index for state in sequence._considered_states] == [0, 22, 33, 44, 99]
 
-        self.assertRaises(SequenceFinished, sequence.next)
+        self.assertRaises(SequenceFinished, lambda: sequence.next(wait=False))
         assert {(first.index, last.index) for (first, last) in sequence._unavailability_gap_pairs} == {(33, 44)}
 
     def test_sbg_search_few_available_search_complex(self):
@@ -73,9 +73,9 @@ def test_sbg_search_few_available_search_complex(self):
 
         while True:
             try:
-                sequence.next()
+                sequence.next(wait=False)
             except SequenceFinished:
                 break
 
-        assert ([state.index for state in sequence._completed_states]
+        assert ([state.index for state in sequence._considered_states]
                 == [0, 12, 22, 34, 36, 38, 44, 56, 66, 68, 72, 78, 88, 98])

test/http_collector/test_collector.py

@@ -10,33 +10,33 @@ class TestBiggestGapBisectionSequence(unittest.TestCase):
     def test_sbg_sequence_always_available(self):
         state_factory = helper.create_state_factory(helper.always_has_binary)
         sequence = BiggestGapBisectionSequence(state_factory, 12)
-        index_sequence = [sequence.next().index for _ in range(12)]
+        index_sequence = [sequence.next(wait=False).index for _ in range(12)]
         assert index_sequence == [0, 99, 49, 74, 24, 36, 61, 86, 12, 42, 67, 92]
-        self.assertRaises(SequenceFinished, sequence.next)
+        self.assertRaises(SequenceFinished, lambda: sequence.next(wait=False))
 
     def test_sbg_sequence_even_available(self):
         state_factory = helper.create_state_factory(helper.only_has_binaries_for_even)
         sequence = BiggestGapBisectionSequence(state_factory, 12)
-        index_sequence = [sequence.next().index for _ in range(12)]
+        index_sequence = [sequence.next(wait=False).index for _ in range(12)]
         assert index_sequence == [0, 98, 48, 72, 24, 84, 12, 36, 60, 90, 6, 18]
 
     def test_sbg_sequence_almost_none_available(self):
         state_factory = helper.create_state_factory(helper.has_very_few_binaries)
         sequence = BiggestGapBisectionSequence(state_factory, 10)
-        index_sequence = [sequence.next().index for _ in range(10)]
+        index_sequence = [sequence.next(wait=False).index for _ in range(10)]
         assert index_sequence == [0, 99, 44, 66, 22, 77, 11, 33, 55, 88]
-        self.assertRaises(SequenceFinished, sequence.next)
+        self.assertRaises(SequenceFinished, lambda: sequence.next(wait=False))
 
     def test_sbg_sequence_sparse_first_half_avaiable(self):
         state_factory = helper.create_state_factory(helper.has_very_few_binaries_in_first_half)
         sequence = BiggestGapBisectionSequence(state_factory, 17)
-        index_sequence = [sequence.next().index for _ in range(17)]
+        index_sequence = [sequence.next(wait=False).index for _ in range(17)]
         assert index_sequence == [0, 99, 50, 22, 74, 44, 86, 62, 92, 56, 68, 80, 95, 53, 59, 65, 71]
 
     def test_sbg_sequence_always_available_with_evaluated_states(self):
         state_factory = helper.create_state_factory(helper.always_has_binary, evaluated_indexes=[49, 61])
         sequence = BiggestGapBisectionSequence(state_factory, 17)
-        index_sequence = [sequence.next().index for _ in range(15)]
+        index_sequence = [sequence.next(wait=False).index for _ in range(15)]
         print(index_sequence)
         assert index_sequence == [0, 99, 24, 80, 36, 12, 70, 89, 42, 6, 18, 30, 55, 75, 94]
-        self.assertRaises(SequenceFinished, sequence.next)
+        self.assertRaises(SequenceFinished, lambda: sequence.next(wait=False))