diff --git a/src/mojxml/parse.py b/src/mojxml/parse.py index d6f3c8f..aea59f3 100644 --- a/src/mojxml/parse.py +++ b/src/mojxml/parse.py @@ -186,7 +186,7 @@ def _parse_features( def parse_raw(content: bytes, options: ParseOptions) -> List[Feature]: - """TODO:""" + """Parse raw XML content and get list of features.""" doc = et.fromstring(content, None) # 基本情報を取得 diff --git a/src/mojxml/process/__init__.py b/src/mojxml/process/__init__.py index 8dd11d8..b4eed92 100644 --- a/src/mojxml/process/__init__.py +++ b/src/mojxml/process/__init__.py @@ -22,7 +22,6 @@ def _write_by_fiona( dst_path: Path, driver: Optional[str] = None, ) -> Iterable[Tuple[int, int]]: # (num_files, num_features) - """WIP""" assert fiona, "fiona is not installed" with fiona.open( @@ -47,7 +46,7 @@ def files_to_ogr_file( executor: BaseExecutor, driver: Optional[str] = None, ) -> None: - """WIP""" + """Generate OGR file from given XML/ZIP files.""" features_iter = executor.iter_process(iter_content_xmls(src_paths)) num_files = 0 @@ -67,7 +66,7 @@ def files_to_ogr_file( def files_to_feature_iter( src_paths: List[Path], executor: BaseExecutor ) -> Iterable[Feature]: - """WIP""" + """Iterate features from given XML/ZIP files.""" features_iter = executor.iter_process(iter_content_xmls(src_paths)) for features in features_iter: for feature in features: diff --git a/src/mojxml/process/executor.py b/src/mojxml/process/executor.py index 641892c..9e1df12 100644 --- a/src/mojxml/process/executor.py +++ b/src/mojxml/process/executor.py @@ -12,7 +12,7 @@ class BaseExecutor(metaclass=ABCMeta): """Executor for processing files""" def __init__(self, options: ParseOptions): - """TODO""" + """Initialize""" self.options = options @abstractmethod @@ -20,23 +20,23 @@ def iter_process( self, src_iter: Iterable[bytes], ) -> Iterable[List[Feature]]: - """TODO""" + """Convert XMLs to OGR features""" class WorkerPoolExecutor(BaseExecutor, metaclass=ABCMeta): - """TODO""" + """Executor implemeted with worker pool""" @abstractmethod - def get_executor(self, max_workers: int) -> concurrent.futures.Executor: - """TODO""" + def _get_executor(self, max_workers: int) -> concurrent.futures.Executor: + ... def iter_process( self, src_iter: Iterable[bytes], ) -> Iterable[List[Feature]]: - """TODO""" + """Convert XMLs to OGR features""" max_workers = os.cpu_count() or 1 - with self.get_executor(max_workers=max_workers) as executor: + with self._get_executor(max_workers=max_workers) as executor: futs = [] for src in src_iter: fut = executor.submit(parse_raw, src, self.options) @@ -60,8 +60,7 @@ def iter_process( class ProcessPoolExecutor(WorkerPoolExecutor): """Process paralelly with ProcessPoolExecutor""" - def get_executor(self, max_workers: int) -> concurrent.futures.Executor: - """TODO""" + def _get_executor(self, max_workers: int) -> concurrent.futures.Executor: max_workers = os.cpu_count() or 1 return concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) @@ -69,20 +68,16 @@ def get_executor(self, max_workers: int) -> concurrent.futures.Executor: class ThreadPoolExecutor(WorkerPoolExecutor): """Process paralelly with ThreadPoolExecutor""" - def get_executor(self, max_workers: int) -> concurrent.futures.Executor: - """TODO""" + def _get_executor(self, max_workers: int) -> concurrent.futures.Executor: max_workers = (os.cpu_count() or 1) * 2 return concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) class SingleThreadExecutor(BaseExecutor): - """Process files with normal iterator""" + """Process files with single-thread (normal) iterator""" - def iter_process( - self, - src_iter: Iterable[bytes], - ) -> Iterable[List[Feature]]: - """TODO""" + def iter_process(self, src_iter: Iterable[bytes]) -> Iterable[List[Feature]]: + """Convert XMLs to OGR features""" for src in src_iter: yield parse_raw(src, options=self.options) diff --git a/src/mojxml/reader.py b/src/mojxml/reader.py index b4810a7..a895fe0 100644 --- a/src/mojxml/reader.py +++ b/src/mojxml/reader.py @@ -6,7 +6,7 @@ def iter_content_xmls(src_paths: List[Path]) -> Iterable[bytes]: - """WIP""" + """Iterate XML contents from given zips and xmls""" for src_path in src_paths: src_path = Path(src_path) if src_path.suffix == ".xml": @@ -23,7 +23,7 @@ class MojXMLZipFile(ZipFile): """法務省登記所備付地図データの多段zip圧縮されたアーカイブを扱う""" def iter_xml_contents(self): - """TODO""" + """Iterate XML contents from given zips""" for name in self.namelist(): if name.endswith(".zip"): yield self._extract_xml_content(name[:-4]) @@ -31,7 +31,6 @@ def iter_xml_contents(self): yield self.open(name).read() def _extract_xml_content(self, internal_name: str) -> bytes: - """TODO""" with self.open(internal_name + ".zip") as f: with ZipFile(f) as zf: xml_content = zf.open(internal_name + ".xml").read()