Coverage for src / competitive_verifier / oj / languages / cplusplus_bundle.py: 70%
197 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-10-04 07:40 +0900
1# Feature to bundle C++ source code into a single file
2# This feature is derived from https://github.com/online-judge-tools/verification-helper
3# Some functionality is missing, but I don't have the capacity to develop and support it rigorously, so I don't plan to update it beyond minor improvements.
4# You can implement your own bundling by performing the bundle separately based on `verify_files.json` and then updating `verify_files.json` with the bundled files.
6# Python Version: 3.x
7import contextlib
8import functools
9import json
10import os
11import pathlib
12import re
13import shutil
14from logging import getLogger
15from typing import Any
17from competitive_verifier.exec import command_stdout
19logger = getLogger(__name__)
21BITS_STDCXX_H = "bits/stdc++.h"
23CXX_STANDARD_LIBS = {
24 "algorithm",
25 "any",
26 "array",
27 "atomic",
28 "barrier",
29 "bit",
30 "bitset",
31 "charconv",
32 "chrono",
33 "codecvt",
34 "compare",
35 "complex",
36 "concepts",
37 "condition_variable",
38 "contracts",
39 "coroutine",
40 "debugging",
41 "deque",
42 "exception",
43 "expected",
44 "filesystem",
45 "flat_map",
46 "flat_set",
47 "format",
48 "forward_list",
49 "fstream",
50 "functional",
51 "future",
52 "generator",
53 "initializer_list",
54 "inplace_vector",
55 "iomanip",
56 "ios",
57 "iosfwd",
58 "iostream",
59 "istream",
60 "iterator",
61 "latch",
62 "limits",
63 "list",
64 "locale",
65 "map",
66 "mdspan",
67 "memory",
68 "memory_resource",
69 "meta",
70 "mutex",
71 "new",
72 "numbers",
73 "numeric",
74 "optional",
75 "ostream",
76 "print",
77 "queue",
78 "random",
79 "ranges",
80 "ratio",
81 "regex",
82 "scoped_allocator",
83 "semaphore",
84 "set",
85 "shared_mutex",
86 "source_location",
87 "span",
88 "spanstream",
89 "sstream",
90 "stack",
91 "stacktrace",
92 "stdexcept",
93 "stdfloat",
94 "stop_token",
95 "streambuf",
96 "string",
97 "string_view",
98 "syncstream",
99 "system_error",
100 "text_encoding",
101 "thread",
102 "tuple",
103 "type_traits",
104 "typeindex",
105 "typeinfo",
106 "unordered_map",
107 "unordered_set",
108 "utility",
109 "valarray",
110 "variant",
111 "vector",
112 "version",
113}
115C_STANDARD_LIBS = {
116 "assert.h",
117 "complex.h",
118 "ctype.h",
119 "errno.h",
120 "fenv.h",
121 "float.h",
122 "inttypes.h",
123 "iso646.h",
124 "limits.h",
125 "locale.h",
126 "math.h",
127 "setjmp.h",
128 "signal.h",
129 "stdalign.h",
130 "stdarg.h",
131 "stdatomic.h",
132 "stdbit.h",
133 "stdbool.h",
134 "stdckdint.h",
135 "stddef.h",
136 "stdint.h",
137 "stdio.h",
138 "stdlib.h",
139 "stdnoreturn.h",
140 "string.h",
141 "tgmath.h",
142 "threads.h",
143 "time.h",
144 "uchar.h",
145 "wchar.h",
146 "wctype.h",
147}
149CXX_C_ORIGIN_LIBS = {"c" + name[: -len(".h")] for name in C_STANDARD_LIBS}
151BITS_EXTCXX_H = "bits/extc++.h"
153EXT_LIBS = {
154 "ext/algorithm",
155 "ext/array_allocator.h",
156 "ext/atomicity.h",
157 "ext/bitmap_allocator.h",
158 "ext/cast.h",
159 "ext/concurrence.h",
160 "ext/debug_allocator.h",
161 "ext/extptr_allocator.h",
162 "ext/functional",
163 "ext/iterator",
164 "ext/malloc_allocator.h",
165 "ext/memory",
166 "ext/mt_allocator.h",
167 "ext/new_allocator.h",
168 "ext/numeric",
169 "ext/pod_char_traits.h",
170 "ext/pointer.h",
171 "ext/pool_allocator.h",
172 "ext/rb_tree",
173 "ext/rope",
174 "ext/slist",
175 "ext/stdio_filebuf.h",
176 "ext/stdio_sync_filebuf.h",
177 "ext/throw_allocator.h",
178 "ext/typelist.h",
179 "ext/type_traits.h",
180 "ext/vstring.h",
181 "ext/pb_ds/assoc_container.hpp",
182 "ext/pb_ds/priority_queue.hpp",
183 "ext/pb_ds/exception.hpp",
184 "ext/pb_ds/hash_policy.hpp",
185 "ext/pb_ds/list_update_policy.hpp",
186 "ext/pb_ds/tree_policy.hpp",
187 "ext/pb_ds/trie_policy.hpp",
188}
190BITS_STDTR1CXX_H = "bits/stdtr1c++.h"
192TR1_LIBS = {
193 "tr1/array",
194 "tr1/cctype",
195 "tr1/cfenv",
196 "tr1/cfloat",
197 "tr1/cinttypes",
198 "tr1/climits",
199 "tr1/cmath",
200 "tr1/complex",
201 "tr1/cstdarg",
202 "tr1/cstdbool",
203 "tr1/cstdint",
204 "tr1/cstdio",
205 "tr1/cstdlib",
206 "tr1/ctgmath",
207 "tr1/ctime",
208 "tr1/cwchar",
209 "tr1/cwctype",
210 "tr1/functional",
211 "tr1/random",
212 "tr1/tuple",
213 "tr1/unordered_map",
214 "tr1/unordered_set",
215 "tr1/utility",
216}
219@functools.cache
220def _check_compiler(compiler: str) -> str:
221 # Executables named "g++" are not always g++, due to the fake g++ of macOS
222 version = command_stdout([compiler, "--version"])
223 if "clang" in version.lower() or "Apple LLVM".lower() in version.lower(): 223 ↛ 224line 223 didn't jump to line 224 because the condition on line 223 was never true
224 return "clang"
225 if "g++" in version.lower(): 225 ↛ 227line 225 didn't jump to line 227 because the condition on line 225 was always true
226 return "gcc"
227 return "unknown" # default
230@functools.cache
231def _get_uncommented_code(
232 path: pathlib.Path, *, iquotes_options: tuple[str, ...], compiler: str
233) -> bytes:
234 # `iquotes_options` must be a tuple to use `lru_cache`
236 if shutil.which(compiler) is None: 236 ↛ 237line 236 didn't jump to line 237 because the condition on line 236 was never true
237 raise BundleError(f"command not found: {compiler}")
238 if _check_compiler(compiler) != "gcc": 238 ↛ 239line 238 didn't jump to line 239 because the condition on line 238 was never true
239 if compiler == "g++":
240 raise BundleError(
241 f"A fake g++ is detected. Please install the GNU C++ compiler.: {compiler}"
242 )
243 raise BundleError(
244 f"It's not g++. Please specify g++ with $CXX envvar.: {compiler}"
245 )
246 command = [
247 compiler,
248 "-x",
249 "c++",
250 *iquotes_options,
251 "-fpreprocessed",
252 "-dD",
253 "-E",
254 str(path),
255 ]
256 return command_stdout(command, text=False)
259def get_uncommented_code(
260 path: pathlib.Path, *, iquotes: list[pathlib.Path], compiler: str
261) -> bytes:
262 iquotes_options: list[str] = []
263 for iquote in iquotes:
264 iquotes_options.extend(["-I", str(iquote.resolve())])
265 code = _get_uncommented_code(
266 path.resolve(), iquotes_options=tuple(iquotes_options), compiler=compiler
267 )
268 lines: list[bytes] = []
269 for line in code.splitlines(keepends=True):
270 m = re.match(rb'# (\d+) ".*"', line.rstrip())
271 if m:
272 lineno = int(m.group(1))
273 while len(lines) + 1 < lineno: 273 ↛ 274line 273 didn't jump to line 274 because the condition on line 273 was never true
274 lines.append(b"\n")
275 else:
276 lines.append(line)
277 return b"".join(lines)
280class BundleError(Exception):
281 pass
284class BundleErrorAt(BundleError): # noqa: N818
285 def __init__(
286 self, path: pathlib.Path, line: int, message: str, *args: Any, **kwargs: Any
287 ):
288 with contextlib.suppress(ValueError):
289 path = path.resolve().relative_to(pathlib.Path.cwd())
290 message = f"{path!s}: line {line}: {message}"
291 super().__init__(message, *args, **kwargs)
294class Bundler:
295 iquotes: list[pathlib.Path]
296 pragma_once: set[pathlib.Path]
297 pragma_once_system: set[str]
298 result_lines: list[bytes]
299 path_stack: set[pathlib.Path]
300 compiler: str
302 def __init__(
303 self,
304 *,
305 iquotes: list[pathlib.Path] | None = None,
306 compiler: str = os.environ.get("CXX", "g++"),
307 ) -> None:
308 if iquotes is None: 308 ↛ 309line 308 didn't jump to line 309 because the condition on line 308 was never true
309 iquotes = []
310 self.iquotes = iquotes
311 self.pragma_once = set()
312 self.pragma_once_system = set()
313 self.result_lines = []
314 self.path_stack = set()
315 self.compiler = compiler
317 # これをしないと __FILE__ や __LINE__ が壊れる
318 def _line(self, line: int, path: pathlib.Path) -> None:
319 while self.result_lines and self.result_lines[-1].startswith(b"#line "):
320 self.result_lines.pop()
321 with contextlib.suppress(ValueError):
322 path = path.relative_to(pathlib.Path.cwd())
323 # パス中の特殊文字を JSON style にエスケープしてから生成コードに記述
324 # quick solution to this: https://github.com/online-judge-tools/verification-helper/issues/280
325 self.result_lines.append(f"#line {line} {json.dumps(str(path))}\n".encode())
327 # path を解決する
328 # see: https://gcc.gnu.org/onlinedocs/gcc/Directory-Options.html#Directory-Options
329 def _resolve(
330 self, path: pathlib.Path, *, included_from: pathlib.Path
331 ) -> pathlib.Path:
332 if (included_from.parent / path).exists(): 332 ↛ 334line 332 didn't jump to line 334 because the condition on line 332 was always true
333 return (included_from.parent / path).resolve()
334 for dir_ in self.iquotes:
335 if (dir_ / path).exists():
336 return (dir_ / path).resolve()
337 raise BundleErrorAt(path, -1, "no such header")
339 def update(self, path: pathlib.Path) -> None:
340 if path.resolve() in self.pragma_once: 340 ↛ 341line 340 didn't jump to line 341 because the condition on line 340 was never true
341 logger.debug(
342 "%s: skipped since this file is included once with include guard",
343 path,
344 )
345 return
347 # 再帰的に自分自身を #include してたら諦める
348 if path in self.path_stack: 348 ↛ 349line 348 didn't jump to line 349 because the condition on line 348 was never true
349 raise BundleErrorAt(path, -1, "cycle found in inclusion relations")
350 self.path_stack.add(path)
351 try:
352 code = path.read_bytes()
353 if not code.endswith(b"\n"): 353 ↛ 355line 353 didn't jump to line 355 because the condition on line 353 was never true
354 # ファイルの末尾に改行がなかったら足す
355 code += b"\n"
357 # include guard のまわりの変数
358 # NOTE: include guard に使われたマクロがそれ以外の用途にも使われたり #undef されたりすると壊れるけど、無視します
359 non_guard_line_found = False
360 pragma_once_found = False
361 include_guard_macro = None
362 include_guard_define_found = False
363 include_guard_endif_found = False
364 preprocess_if_nest = 0
366 lines = code.splitlines(keepends=True)
367 uncommented_lines = get_uncommented_code(
368 path, iquotes=self.iquotes, compiler=self.compiler
369 ).splitlines(keepends=True)
370 uncommented_lines.extend(
371 [b""] * (len(lines) - len(uncommented_lines))
372 ) # trailing comment lines are removed
373 assert len(lines) == len(uncommented_lines)
374 self._line(1, path)
375 for i, (line, uncommented_line) in enumerate(
376 zip(lines, uncommented_lines, strict=False)
377 ):
378 # nest の処理
379 if re.match(rb"\s*#\s*(if|ifdef|ifndef)\s.*", uncommented_line):
380 preprocess_if_nest += 1
381 if ( 381 ↛ 385line 381 didn't jump to line 385 because the condition on line 381 was never true
382 re.match(rb"\s*#\s*(else\s*|elif\s.*)", uncommented_line)
383 and preprocess_if_nest == 0
384 ):
385 raise BundleErrorAt(path, i + 1, "unmatched #else / #elif")
386 if re.match(rb"\s*#\s*endif\s*", uncommented_line):
387 preprocess_if_nest -= 1
388 if preprocess_if_nest < 0: 388 ↛ 389line 388 didn't jump to line 389 because the condition on line 388 was never true
389 raise BundleErrorAt(path, i + 1, "unmatched #endif")
390 is_toplevel = preprocess_if_nest == 0 or (
391 preprocess_if_nest == 1 and include_guard_macro is not None
392 )
394 # #pragma once
395 if re.match(
396 rb"\s*#\s*pragma\s+once\s*", line
397 ): # #pragma once は comment 扱いで消されてしまう
398 logger.debug("%s: line %s: #pragma once", path, i + 1)
399 if non_guard_line_found: 399 ↛ 401line 399 didn't jump to line 401 because the condition on line 399 was never true
400 # 先頭以外で #pragma once されてた場合は諦める
401 raise BundleErrorAt(
402 path, i + 1, "#pragma once found in a non-first line"
403 )
404 if include_guard_macro is not None: 404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true
405 raise BundleErrorAt(
406 path,
407 i + 1,
408 "#pragma once found in an include guard with #ifndef",
409 )
410 if path.resolve() in self.pragma_once: 410 ↛ 411line 410 didn't jump to line 411 because the condition on line 410 was never true
411 return
412 pragma_once_found = True
413 self.pragma_once.add(path.resolve())
414 self._line(i + 2, path)
415 continue
417 matched: re.Match[bytes] | None
418 # #ifndef HOGE_H as guard
419 if (
420 not pragma_once_found
421 and not non_guard_line_found
422 and include_guard_macro is None
423 ):
424 matched = re.match(rb"\s*#\s*ifndef\s+(\w+)\s*", uncommented_line)
425 if matched:
426 include_guard_macro = matched.group(1).decode()
427 logger.debug(
428 "%s: line %s: #ifndef %s",
429 path,
430 i + 1,
431 include_guard_macro,
432 )
433 self.result_lines.append(b"\n")
434 continue
436 # #define HOGE_H as guard
437 if include_guard_macro is not None and not include_guard_define_found:
438 matched = re.match(rb"\s*#\s*define\s+(\w+)\s*", uncommented_line)
439 if matched and matched.group(1).decode() == include_guard_macro: 439 ↛ 452line 439 didn't jump to line 452 because the condition on line 439 was always true
440 self.pragma_once.add(path.resolve())
441 logger.debug(
442 "%s: line %s: #define %s",
443 path,
444 i + 1,
445 include_guard_macro,
446 )
447 include_guard_define_found = True
448 self.result_lines.append(b"\n")
449 continue
451 # #endif as guard
452 if (
453 include_guard_define_found
454 and preprocess_if_nest == 0
455 and not include_guard_endif_found
456 ) and re.match(rb"\s*#\s*endif\s*", uncommented_line):
457 include_guard_endif_found = True
458 self.result_lines.append(b"\n")
459 continue
461 if uncommented_line and not re.match(rb"^\s*$", uncommented_line):
462 non_guard_line_found = True
463 if ( 463 ↛ 468line 463 didn't jump to line 468 because the condition on line 463 was never true
464 include_guard_macro is not None
465 and not include_guard_define_found
466 ):
467 # 先頭に #ifndef が見付かっても #define が続かないならそれは include guard ではない
468 include_guard_macro = None
469 if include_guard_endif_found: 469 ↛ 471line 469 didn't jump to line 471 because the condition on line 469 was never true
470 # include guard の外側にコードが書かれているとまずいので検出する
471 raise BundleErrorAt(
472 path, i + 1, "found codes out of include guard"
473 )
475 # #include <...>
476 matched = re.match(rb"\s*#\s*include\s*<(.*)>\s*", uncommented_line)
477 if matched:
478 included = matched.group(1).decode()
479 logger.debug(
480 "%s: line %s: #include <%s>", path, i + 1, str(included)
481 )
482 if included in self.pragma_once_system: 482 ↛ 483line 482 didn't jump to line 483 because the condition on line 482 was never true
483 self._line(i + 2, path)
484 elif not is_toplevel: 484 ↛ 486line 484 didn't jump to line 486 because the condition on line 484 was never true
485 # #pragma once 系の判断ができない場合はそっとしておく
486 self.result_lines.append(line)
487 elif ( 487 ↛ 497line 487 didn't jump to line 497 because the condition on line 487 was always true
488 included in C_STANDARD_LIBS
489 or included in CXX_STANDARD_LIBS
490 or included in CXX_C_ORIGIN_LIBS
491 ):
492 if BITS_STDCXX_H in self.pragma_once_system: 492 ↛ 493line 492 didn't jump to line 493 because the condition on line 492 was never true
493 self._line(i + 2, path)
494 else:
495 self.pragma_once_system.add(included)
496 self.result_lines.append(line)
497 elif included in EXT_LIBS:
498 if BITS_EXTCXX_H in self.pragma_once_system:
499 self._line(i + 2, path)
500 else:
501 self.pragma_once_system.add(included)
502 self.result_lines.append(line)
503 elif included in TR1_LIBS:
504 if BITS_STDTR1CXX_H in self.pragma_once_system:
505 self._line(i + 2, path)
506 else:
507 self.pragma_once_system.add(included)
508 self.result_lines.append(line)
509 else:
510 # possibly: bits/*, tr2/* boost/*, c-posix library, etc.
511 self.pragma_once_system.add(included)
512 self.result_lines.append(line)
513 if included in [BITS_EXTCXX_H, BITS_STDTR1CXX_H]:
514 self.pragma_once_system.add(BITS_STDCXX_H)
515 continue
517 # #include "..."
518 matched = re.match(rb'\s*#\s*include\s*"(.*)"\s*', uncommented_line)
519 if matched:
520 included = matched.group(1).decode()
521 logger.debug('%s: line %s: #include "%s"', path, i + 1, included)
522 if not is_toplevel: 522 ↛ 524line 522 didn't jump to line 524 because the condition on line 522 was never true
523 # #if の中から #include されると #pragma once 系の判断が不可能になるので諦める
524 raise BundleErrorAt(
525 path,
526 i + 1,
527 "unable to process #include in #if / #ifdef / #ifndef other than include guards",
528 )
529 self.update(
530 self._resolve(pathlib.Path(included), included_from=path)
531 )
532 self._line(i + 2, path)
533 # #include "iostream" みたいに書いたときの挙動をはっきりさせる
534 # #include <iostream> /* とかをやられた場合を落とす
535 continue
537 # otherwise
538 self.result_lines.append(line)
540 # #if #endif の対応が壊れてたら諦める
541 last_index = i + 1 # pyright: ignore[reportPossiblyUnboundVariable]
543 if preprocess_if_nest != 0: 543 ↛ 544line 543 didn't jump to line 544 because the condition on line 543 was never true
544 raise BundleErrorAt(
545 path, last_index, "unmatched #if / #ifdef / #ifndef"
546 )
547 if include_guard_macro is not None and not include_guard_endif_found: 547 ↛ 548line 547 didn't jump to line 548 because the condition on line 547 was never true
548 raise BundleErrorAt(path, last_index, "unmatched #ifndef")
550 finally:
551 # 中で return することがあるので finally 節に入れておく
552 self.path_stack.remove(path)
554 def get(self) -> bytes:
555 return b"".join(self.result_lines)