Skip to content

Commit

Permalink
Update code comments
Browse files Browse the repository at this point in the history
  • Loading branch information
ciscorn committed Mar 17, 2023
1 parent ec6d6f7 commit ae8fb11
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 24 deletions.
2 changes: 1 addition & 1 deletion src/mojxml/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ def _parse_features(


def parse_raw(content: bytes, options: ParseOptions) -> List[Feature]:
"""TODO:"""
"""Parse raw XML content and get list of features."""
doc = et.fromstring(content, None)

# 基本情報を取得
Expand Down
5 changes: 2 additions & 3 deletions src/mojxml/process/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ def _write_by_fiona(
dst_path: Path,
driver: Optional[str] = None,
) -> Iterable[Tuple[int, int]]: # (num_files, num_features)
"""WIP"""
assert fiona, "fiona is not installed"

with fiona.open(
Expand All @@ -47,7 +46,7 @@ def files_to_ogr_file(
executor: BaseExecutor,
driver: Optional[str] = None,
) -> None:
"""WIP"""
"""Generate OGR file from given XML/ZIP files."""
features_iter = executor.iter_process(iter_content_xmls(src_paths))

num_files = 0
Expand All @@ -67,7 +66,7 @@ def files_to_ogr_file(
def files_to_feature_iter(
src_paths: List[Path], executor: BaseExecutor
) -> Iterable[Feature]:
"""WIP"""
"""Iterate features from given XML/ZIP files."""
features_iter = executor.iter_process(iter_content_xmls(src_paths))
for features in features_iter:
for feature in features:
Expand Down
29 changes: 12 additions & 17 deletions src/mojxml/process/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,31 @@ class BaseExecutor(metaclass=ABCMeta):
"""Executor for processing files"""

def __init__(self, options: ParseOptions):
"""TODO"""
"""Initialize"""
self.options = options

@abstractmethod
def iter_process(
self,
src_iter: Iterable[bytes],
) -> Iterable[List[Feature]]:
"""TODO"""
"""Convert XMLs to OGR features"""


class WorkerPoolExecutor(BaseExecutor, metaclass=ABCMeta):
"""TODO"""
"""Executor implemeted with worker pool"""

@abstractmethod
def get_executor(self, max_workers: int) -> concurrent.futures.Executor:
"""TODO"""
def _get_executor(self, max_workers: int) -> concurrent.futures.Executor:
...

def iter_process(
self,
src_iter: Iterable[bytes],
) -> Iterable[List[Feature]]:
"""TODO"""
"""Convert XMLs to OGR features"""
max_workers = os.cpu_count() or 1
with self.get_executor(max_workers=max_workers) as executor:
with self._get_executor(max_workers=max_workers) as executor:
futs = []
for src in src_iter:
fut = executor.submit(parse_raw, src, self.options)
Expand All @@ -60,29 +60,24 @@ def iter_process(
class ProcessPoolExecutor(WorkerPoolExecutor):
"""Process paralelly with ProcessPoolExecutor"""

def get_executor(self, max_workers: int) -> concurrent.futures.Executor:
"""TODO"""
def _get_executor(self, max_workers: int) -> concurrent.futures.Executor:
max_workers = os.cpu_count() or 1
return concurrent.futures.ProcessPoolExecutor(max_workers=max_workers)


class ThreadPoolExecutor(WorkerPoolExecutor):
"""Process paralelly with ThreadPoolExecutor"""

def get_executor(self, max_workers: int) -> concurrent.futures.Executor:
"""TODO"""
def _get_executor(self, max_workers: int) -> concurrent.futures.Executor:
max_workers = (os.cpu_count() or 1) * 2
return concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)


class SingleThreadExecutor(BaseExecutor):
"""Process files with normal iterator"""
"""Process files with single-thread (normal) iterator"""

def iter_process(
self,
src_iter: Iterable[bytes],
) -> Iterable[List[Feature]]:
"""TODO"""
def iter_process(self, src_iter: Iterable[bytes]) -> Iterable[List[Feature]]:
"""Convert XMLs to OGR features"""
for src in src_iter:
yield parse_raw(src, options=self.options)

Expand Down
5 changes: 2 additions & 3 deletions src/mojxml/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@


def iter_content_xmls(src_paths: List[Path]) -> Iterable[bytes]:
"""WIP"""
"""Iterate XML contents from given zips and xmls"""
for src_path in src_paths:
src_path = Path(src_path)
if src_path.suffix == ".xml":
Expand All @@ -23,15 +23,14 @@ class MojXMLZipFile(ZipFile):
"""法務省登記所備付地図データの多段zip圧縮されたアーカイブを扱う"""

def iter_xml_contents(self):
"""TODO"""
"""Iterate XML contents from given zips"""
for name in self.namelist():
if name.endswith(".zip"):
yield self._extract_xml_content(name[:-4])
elif name.endswith(".xml"):
yield self.open(name).read()

def _extract_xml_content(self, internal_name: str) -> bytes:
"""TODO"""
with self.open(internal_name + ".zip") as f:
with ZipFile(f) as zf:
xml_content = zf.open(internal_name + ".xml").read()
Expand Down

0 comments on commit ae8fb11

Please sign in to comment.