diff --git a/droidasc/asc_client/apk_handler.py b/droidasc/asc_client/apk_handler.py index 864aaea..f943455 100644 --- a/droidasc/asc_client/apk_handler.py +++ b/droidasc/asc_client/apk_handler.py @@ -359,7 +359,19 @@ def _open_apk(self): return fp, mm def get_class_dex(self, dalvik_class : str): + # string_data_item is MUTF-8, not UTF-8: a code point above the BMP is + # stored as a surrogate pair (CESU-8), so a UTF-8-encoded query can never + # byte-match it and such a class was unreachable. MUTF-8 also keeps byte + # order aligned with the table's UTF-16 code-unit order, which the binary + # search in _find_type_idx relies on. + # + # The two encodings differ only above the BMP (and for an embedded NUL, + # which a descriptor cannot hold), so only such a name pays for the encoder + # and an ordinary lookup stays on the plain UTF-8 path. target_bytes = dalvik_class.encode("utf-8") + if any(ord(ch) > 0xFFFF for ch in dalvik_class): + from mutf8 import encode_modified_utf8 + target_bytes = encode_modified_utf8(dalvik_class) t_start = time.perf_counter() fp, mm = self._open_apk() try: diff --git a/droidasc/asc_client/asc_handler.py b/droidasc/asc_client/asc_handler.py index 63435ce..69cb513 100644 --- a/droidasc/asc_client/asc_handler.py +++ b/droidasc/asc_client/asc_handler.py @@ -70,20 +70,28 @@ def getclass_with_metadata(self, dex_buf : bytes, dalvik_class : str): new_dex_bytes = manager.extract_and_rebuild(dalvik_class) return _decompile_dex_bytes_with_metadata(new_dex_bytes, dalvik_class) - def _format_method(self, dex, midx : int) -> str: + def _format_method(self, dex, midx : int, include_descriptor : bool = False) -> str: method = dex.methods[midx] - return f"{method.cls.fullname}->{method.name}" - - def _format_matched_name(self, dex, find_type : str, idx : int) -> str: + ret = f"{method.cls.fullname}->{method.name}" + if not include_descriptor: + return ret + proto = method.prototype + descriptor = "(" + "".join(item.descriptor for item in proto.parameters_type) + ")" + descriptor += dex.get_type(proto.return_type_idx).descriptor + return ret + descriptor + + def _format_matched_name( + self, dex, find_type : str, idx : int, include_descriptor : bool = False, + ) -> str: if find_type == "string": return str(dex.strings[idx]) if find_type == "type": return dex.types[idx].descriptor if find_type == "method": - method = dex.methods[idx] - return f"{method.cls.fullname}->{method.name}" + return self._format_method(dex, idx, include_descriptor=include_descriptor) field = dex.fields[idx] - return f"{field.cls.fullname}->{field.name}" + ret = f"{field.cls.fullname}->{field.name}" + return f"{ret}:{field.type.descriptor}" if include_descriptor else ret def findrefs(self, dex_name : str, dex_buf : bytes, find_type : str, find : dict, aggregate : bool = True) -> list: from droidasc.asc_core.findrefs.findrefs_manager import FindRefManager @@ -96,6 +104,7 @@ def findrefs(self, dex_name : str, dex_buf : bytes, find_type : str, find : dict mids = query[find_type] if not aggregate: ret = [] + occurrences = defaultdict(int) for i in range(len(mids)): midx = mids[i] if midx is None: @@ -105,10 +114,16 @@ def findrefs(self, dex_name : str, dex_buf : bytes, find_type : str, find : dict midxs = midx else: midxs = [midx] - matched = self._format_matched_name(dex, find_type, idx) + matched = self._format_matched_name( + dex, find_type, idx, include_descriptor=True + ) for mid in midxs: + key = (mid, idx) + occurrence = occurrences[key] + occurrences[key] += 1 ret.append( - f"{dex_name} | {self._format_method(dex, mid)} | matched=({matched})" + f"{dex_name} | {self._format_method(dex, mid, include_descriptor=True)} | " + f"matched=({matched}) | occurrence={occurrence}" ) return ret diff --git a/droidasc/asc_client/gui/app.py b/droidasc/asc_client/gui/app.py index 6ad8fe7..d39c405 100644 --- a/droidasc/asc_client/gui/app.py +++ b/droidasc/asc_client/gui/app.py @@ -14,6 +14,7 @@ index_to_offset, is_identifier, find_member_declaration, + find_search_reference, linkable_member_spans, member_reference_at_offset, remap_ranges_after_replacements, @@ -172,7 +173,7 @@ def _open_selected(self, _event = None): if not selection: return row = self.rows[int(selection[0])] - self.app.open_class(row["class_name"]) + self.app.open_search_result(row, self.find_type) def _append_visible_rows(self, rows): if not rows: @@ -841,6 +842,45 @@ def _complete_pending_member_navigation(self, dalvik_class : str): tab = self.editor_tabs[idx] if tab.loading or tab.error or not tab.source: return + member_type = target[1] + if member_type == "search": + ( + _class_name, _kind, method_name, method_descriptor, + target_type, target_class, target_name, target_descriptor, occurrence, + ) = target + source = tab.edited_source or tab.source + references = tab.rendered_member_references + if tab.comments: + source, _comment_spans = self._render_tab_source(tab) + references = tab.rendered_member_references + reference = find_search_reference( + source, references, method_name, method_descriptor, + target_type, target_class, target_name, target_descriptor, occurrence, + ) + exact = reference is not None + if reference is None: + reference = find_member_declaration( + references, "method", method_name, method_descriptor + ) + if reference is None: + reference = find_member_declaration(references, "method", method_name, "") + self._pending_member_navigation = None + if reference is None: + self.status_var.set(f"Search location not found: {dalvik_to_dot(dalvik_class)}->{method_name}") + return + start = reference[0] + index = f"1.0+{start}c" + self.source_text.mark_set("insert", index) + self.source_text.see(index) + tab.insert_index = self.source_text.index("insert") + tab.yview = self.source_text.yview() + self._highlight_active_line() + self._highlight_related_identifier(tab.insert_index) + suffix = "" if exact else " (exact expression unavailable)" + self.status_var.set( + f"Opened search hit {dalvik_to_dot(dalvik_class)}->{method_name}{suffix}" + ) + return _class_name, member_type, member_name, descriptor = target reference = find_member_declaration( tab.rendered_member_references, member_type, member_name, descriptor @@ -857,7 +897,10 @@ def _complete_pending_member_navigation(self, dalvik_class : str): tab.yview = self.source_text.yview() self._highlight_active_line() self._highlight_related_identifier(tab.insert_index) - self.status_var.set(f"Opened declaration {dalvik_to_dot(dalvik_class)}->{member_name}") + if member_type == "class": + self.status_var.set(f"Opened class {dalvik_to_dot(dalvik_class)}") + else: + self.status_var.set(f"Opened declaration {dalvik_to_dot(dalvik_class)}->{member_name}") def _on_source_key_press(self, event): if event.state & 0x0004: @@ -892,6 +935,9 @@ def _find_current_member_references(self, _event = None): self.status_var.set("Place the cursor on a method or field name to find its references") return "break" _start, _end, member_type, class_name, member_name, *_metadata = reference + if member_type not in ("method", "field"): + self.status_var.set("Place the cursor on a method or field name to find its references") + return "break" self.search_type_var.set(f"{member_type} refs") self.search_value_var.set(member_name) @@ -1641,6 +1687,24 @@ def worker(): threading.Thread(target=worker, daemon=True).start() + def open_search_result(self, row, find_type : str): + class_name = row["class_name"] + target_type = row.get("target_type", "") + if find_type in ("method", "field"): + self._pending_member_navigation = ( + class_name, target_type, row.get("target_name", ""), + row.get("target_descriptor", ""), + ) + else: + self._pending_member_navigation = ( + class_name, "search", row.get("method_name", ""), + row.get("method_descriptor", ""), target_type, + row.get("target_class", ""), row.get("target_name", ""), + row.get("target_descriptor", ""), row.get("occurrence", 0), + ) + self.open_class(class_name) + self._complete_pending_member_navigation(class_name) + def _start_search(self, exact_member : bool = False): if self.store is None: return diff --git a/droidasc/asc_client/gui/runtime.py b/droidasc/asc_client/gui/runtime.py index 3047061..f0262c3 100644 --- a/droidasc/asc_client/gui/runtime.py +++ b/droidasc/asc_client/gui/runtime.py @@ -101,15 +101,47 @@ def build_find_query( return find_type, {find_type: {"class": [class_name, not fuzzy_class], find_type: member}} -def parse_result_line(line : str): +def _split_member_text(text : str, member_type : str): + class_name, member_text = text.split("->", 1) + if member_type == "method": + split_at = member_text.find("(") + else: + split_at = member_text.rfind(":") + if split_at < 0: + return class_name, member_text, "" + if member_type == "method": + return class_name, member_text[:split_at], member_text[split_at:] + return class_name, member_text[:split_at], member_text[split_at + 1:] + + +def parse_result_line(line : str, find_type : str = ""): dex_name, method_text, matched_text = line.split(" | ", 2) - class_name = method_text.split("->", 1)[0] + matched_text, sep, occurrence_text = matched_text.rpartition(" | occurrence=") + occurrence = int(occurrence_text) if sep and occurrence_text.isdigit() else 0 + if not sep: + matched_text = occurrence_text + class_name, method_name, method_descriptor = _split_member_text(method_text, "method") + matched_value = matched_text[len("matched=("):-1] if matched_text.startswith("matched=(") else matched_text + target_class = "" + target_name = matched_value + target_descriptor = "" + if find_type in ("method", "field"): + target_class, target_name, target_descriptor = _split_member_text(matched_value, find_type) + elif find_type == "type": + target_class = matched_value return { "dex_name": dex_name, "class_name": class_name, "class_display": dalvik_to_dot(class_name), "method_text": method_text, + "method_name": method_name, + "method_descriptor": method_descriptor, "matched_text": matched_text, + "target_type": "class" if find_type == "type" else find_type, + "target_class": target_class, + "target_name": target_name, + "target_descriptor": target_descriptor, + "occurrence": occurrence, "line": line, } @@ -296,13 +328,23 @@ def search_dex(dex_name): for midx in sorted(member_idxs): method = dex.methods[midx] class_name = method.cls.fullname - method_text = f"{class_name}->{method.name}" + proto = method.prototype + descriptor = "(" + "".join(item.descriptor for item in proto.parameters_type) + ")" + descriptor += dex.get_type(proto.return_type_idx).descriptor + method_text = f"{class_name}->{method.name}{descriptor}" dex_rows.append({ "dex_name": dex_name, "class_name": class_name, "class_display": dalvik_to_dot(class_name), "method_text": method_text, + "method_name": method.name, + "method_descriptor": descriptor, "matched_text": f"method=({method.name})", + "target_type": "method", + "target_class": class_name, + "target_name": method.name, + "target_descriptor": descriptor, + "occurrence": 0, "line": method_text, }) return len(member_idxs), dex_rows @@ -318,7 +360,14 @@ def search_dex(dex_name): "class_name": class_name, "class_display": dalvik_to_dot(class_name), "method_text": method_text, + "method_name": field.name, + "method_descriptor": "", "matched_text": f"field=({field.name})", + "target_type": "field", + "target_class": class_name, + "target_name": field.name, + "target_descriptor": field.type.descriptor, + "occurrence": 0, "line": method_text, }) return len(member_idxs), dex_rows @@ -406,7 +455,7 @@ def _search_with_process_pool(self, executor, workers, find_type, find, progress if len(shown) < _MAX_UI_RESULTS: remain = _MAX_UI_RESULTS - len(shown) for line in lines[:remain]: - row = parse_result_line(line) + row = parse_result_line(line, find_type) shown.append(row) batch_rows.append(row) if batch_rows and result_callback is not None: @@ -495,7 +544,7 @@ def _search_with_subprocess_pool(self, workers, find_type, find, progress_callba if len(shown) < _MAX_UI_RESULTS: remain = _MAX_UI_RESULTS - len(shown) for line in lines[:remain]: - row = parse_result_line(line) + row = parse_result_line(line, find_type) shown.append(row) batch_rows.append(row) if batch_rows and result_callback is not None: diff --git a/droidasc/asc_client/gui/source_edit.py b/droidasc/asc_client/gui/source_edit.py index e916396..6f126ee 100644 --- a/droidasc/asc_client/gui/source_edit.py +++ b/droidasc/asc_client/gui/source_edit.py @@ -280,6 +280,43 @@ def find_member_declaration(references, member_type : str, member_name : str, de return None +def find_search_reference( + source : str, references, method_name : str, method_descriptor : str, + target_type : str, target_class : str, target_name : str, + target_descriptor : str, occurrence : int = 0, +): + declaration = find_member_declaration( + references, "method", method_name, method_descriptor + ) + if declaration is None: + declaration = find_member_declaration(references, "method", method_name, "") + if declaration is None: + return None + method_range = find_method_range(source, declaration[0]) + if method_range is None: + return None + + def matches(item): + if not (method_range[0] <= item[0] < method_range[1]) or item[6]: + return False + if item[2] != target_type: + return False + if target_type == "string": + return item[4] == target_name + if target_type == "class": + return item[3] == target_class + return ( + item[3] == target_class + and item[4] == target_name + and (not target_descriptor or item[5] == target_descriptor) + ) + + matches_in_method = [item for item in references if matches(item)] + if occurrence < 0 or occurrence >= len(matches_in_method): + return None + return matches_in_method[occurrence] + + def linkable_member_spans(references, available_classes): return [ (item[0], item[1]) diff --git a/droidasc/asc_core/findrefs/locator/field_locator.py b/droidasc/asc_core/findrefs/locator/field_locator.py index 4192746..4a6bcf5 100644 --- a/droidasc/asc_core/findrefs/locator/field_locator.py +++ b/droidasc/asc_core/findrefs/locator/field_locator.py @@ -48,6 +48,9 @@ def _find_type_idx_precisely(self, clz : str): right = type_ids_size - 1 buf = self.buf strings = self.dex.strings + # see MethodLocator._find_type_idx_precisely: type_ids is sorted by UTF-16 + # code unit, so the search has to compare in that order, not Python's + key = clz.encode('utf-16-be', 'surrogatepass') while left <= right: mid = (left + right) >> 1 @@ -55,7 +58,7 @@ def _find_type_idx_precisely(self, clz : str): desc = strings[desc_idx] if desc == clz: return mid - if desc < clz: + if desc.encode('utf-16-be', 'surrogatepass') < key: left = mid + 1 else: right = mid - 1 diff --git a/droidasc/asc_core/findrefs/locator/method_locator.py b/droidasc/asc_core/findrefs/locator/method_locator.py index bc9b7c8..f46bcfe 100644 --- a/droidasc/asc_core/findrefs/locator/method_locator.py +++ b/droidasc/asc_core/findrefs/locator/method_locator.py @@ -47,6 +47,13 @@ def _find_type_idx_precisely(self, clz : str): right = type_ids_size - 1 buf = self.buf strings = self.dex.strings + # type_ids is sorted by UTF-16 code unit, which is not Python's order above + # the BMP: a surrogate pair is 0xD800-0xDFFF, so U+10000 sorts before + # U+E000, while Python compares those two the other way round. Comparing + # string values prunes the wrong half and misses entries; comparing the + # UTF-16BE encoding compares in the table's own order. surrogatepass because + # a malformed dex can decode to lone surrogates, which utf-16 refuses. + key = clz.encode('utf-16-be', 'surrogatepass') while left <= right: mid = (left + right) >> 1 @@ -54,7 +61,7 @@ def _find_type_idx_precisely(self, clz : str): desc = strings[desc_idx] if desc == clz: return mid - if desc < clz: + if desc.encode('utf-16-be', 'surrogatepass') < key: left = mid + 1 else: right = mid - 1 diff --git a/droidasc/asc_core/findrefs/locator/string_locator.py b/droidasc/asc_core/findrefs/locator/string_locator.py index f410f92..c42f6de 100644 --- a/droidasc/asc_core/findrefs/locator/string_locator.py +++ b/droidasc/asc_core/findrefs/locator/string_locator.py @@ -46,13 +46,20 @@ def _build_map(self): def _match_string_offset(self, string : str): buf = self.buf - string = string.encode('utf-8') + # the region below holds MUTF-8, where a code point above the BMP is stored + # as a surrogate pair (CESU-8) that a UTF-8 pattern can never match. The two + # encodings agree below the BMP, so only such a pattern pays for the encoder + # and an ordinary search stays on the plain UTF-8 path. + pattern_bytes = string.encode('utf-8') + if any(ord(ch) > 0xFFFF for ch in string): + from mutf8 import encode_modified_utf8 + pattern_bytes = encode_modified_utf8(string) strdata_start = self.strdata_start strdata_end = self.strdata_end submem = buf[strdata_start: strdata_end] mm = submem.obj - pattern = re.compile(string) + pattern = re.compile(pattern_bytes) # offsets = [] for match in pattern.finditer(submem): diff --git a/droidasc/asc_core/utils/decompiler.py b/droidasc/asc_core/utils/decompiler.py index fcbfee4..38cd7fa 100644 --- a/droidasc/asc_core/utils/decompiler.py +++ b/droidasc/asc_core/utils/decompiler.py @@ -16,6 +16,7 @@ def monkey_header_init(self, offset, buff, cm): from dadguard.decompiler import decompile from dadguard.decompiler import instruction as dadguard_instruction from dadguard.decompiler import util as dadguard_util +from dadguard.decompiler import writer as dadguard_writer from dadguard.core.analysis.analysis import MethodAnalysis import dadguard.core.androconf as androconf import functools @@ -199,12 +200,27 @@ def patched_static_instruction_visit(self, visitor): ) +_orig_writer_visit_constant = decompile.Writer.visit_constant +def patched_writer_visit_constant(self, cst): + if not isinstance(cst, str): + return _orig_writer_visit_constant(self, cst) + value = dadguard_writer.string(cst) + self.write(value) + self.write_ext(("CONSTANT_STRING", value, cst)) + + +def patched_base_class_visit(self, visitor): + return visitor.visit_base_class(self.cls, data=self.clsdesc or self.cls) + + dadguard_instruction.InstanceExpression.visit = patched_instance_expression_visit dadguard_instruction.InstanceInstruction.visit = patched_instance_instruction_visit dadguard_instruction.StaticExpression.visit = patched_static_expression_visit dadguard_instruction.StaticInstruction.visit = patched_static_instruction_visit +dadguard_instruction.BaseClass.visit = patched_base_class_visit decompile.Writer.visit_get_static = patched_writer_get_static decompile.Writer.visit_put_static = patched_writer_put_static +decompile.Writer.visit_constant = patched_writer_visit_constant # --- End of String/Type Optimizations --- # We also completely disable ALL dadguard loggers via python's standard logging module @@ -273,6 +289,20 @@ def _dalvik_class_name(name : str) -> str: return name +def _dalvik_reference_class(name : str): + name = (name or "").strip() + while name.startswith("["): + name = name[1:] + while name.endswith("[]"): + name = name[:-2] + if not name or name in { + "V", "Z", "B", "S", "C", "I", "J", "F", "D", + "void", "boolean", "byte", "short", "char", "int", "long", "float", "double", + }: + return None + return _dalvik_class_name(name) + + # reuse dadguard daddecompiler's token type, we can extract source' semantics, enhance gui xref ability # 20260921 we may extract daddecompiler from dadguard in future, for better development def _source_with_member_references(source_ext): @@ -321,6 +351,35 @@ def append_tokens(tokens): field.get_descriptor(), True, )) + elif kind == "CONSTANT_STRING": + raw_value = token[2] if len(token) >= 3 else value[1:-1] + references.append((start, offset, "string", "", raw_value, "", False)) + elif kind in ("CHECKCAST", "NEW_ARRAY", "NEW_ARRAY_FILLED"): + type_start = 2 if kind == "CHECKCAST" else 4 + type_end = value.find(") ", type_start) if kind == "CHECKCAST" else value.rfind("[") + if kind == "NEW_ARRAY_FILLED": + type_end = value.rfind(" {") + class_name = _dalvik_reference_class(value[type_start:type_end]) + if class_name is not None: + references.append(( + start + type_start, start + type_end, + "class", class_name, "", "", False, + )) + elif kind in ( + "VARIABLE_TYPE", "EXCEPTION_TYPE", + "FIELD_TYPE", "ARG_TYPE", "PROTOTYPE_TYPE", + "NAME_SUPERCLASS", "NAME_INTERFACE", "NAME_PROTOTYPE", + "NAME_BASE_CLASS", "NAME_CLASS_NEW", + ): + raw_name = token[2] if len(token) >= 3 else value + declaration = kind == "NAME_PROTOTYPE" + if declaration and len(token) >= 3 and token[2]: + raw_name = f"{token[2]}.{value}" + class_name = _dalvik_reference_class(raw_name) + if class_name is not None: + references.append(( + start, offset, "class", class_name, "", "", declaration, + )) elif kind in ("NAME_CLASS_INSTANCE", "NAME_CLASS_ASSIGNMENT") and len(token) >= 3: field = token[2] references.append(( @@ -333,9 +392,14 @@ def append_tokens(tokens): elif kind in ("GET_STATIC", "PUT_STATIC") and len(token) >= 3: field = token[2] field_start = offset - len(field.name) + class_name = _dalvik_class_name(field.clsdesc) + if field_start > start: + references.append(( + start, field_start - 1, "class", class_name, "", "", False, + )) references.append(( field_start, offset, "field", - _dalvik_class_name(field.clsdesc), + class_name, field.name, field.ftype, False, diff --git a/tests/test_references.py b/tests/test_references.py index f607114..cef4548 100644 --- a/tests/test_references.py +++ b/tests/test_references.py @@ -155,6 +155,15 @@ def test_empty_string_table(self): struct.pack_into('