Merge pull request #26 from farzai/feature/memory-optimization

Replace Guzzle with farzai/transport and optimize memory

Author: parsilver

README.md

@@ -14,7 +14,17 @@ A PHP library for downloading and converting Geonames data. This library provide
 - Import data directly to MongoDB with proper indexing
 - Memory-efficient processing for large datasets
 - Progress bars for all operations
-- Support for filtering by feature types (for Gazetteer data)
+
+## How It Works
+
+### Admin Code Resolution
+When downloading gazetteer data, the library automatically downloads `admin1CodesASCII.txt` and `admin2Codes.txt` from GeoNames to resolve administrative division names (e.g., converting admin code "40" to "Bangkok").
+
+### Memory-Efficient Processing
+Data is processed using streaming to handle large datasets without exhausting memory. For MongoDB imports, records are inserted in batches of 1000 for optimal performance.
+
+### Automatic Cleanup
+Temporary files (downloaded ZIP files, extracted data files, and admin code files) are automatically cleaned up after processing completes.
 
 ## Installation
 
@@ -70,22 +80,10 @@ Download geographical data for all countries:
 Options:
 - `--output (-o)`: Output directory (default: ./data)
 - `--format (-f)`: Output format (default: json, options: json, mongodb)
-- `--feature-class (-c)`: Filter by feature class (default: P)
 - `--mongodb-uri`: MongoDB connection URI (default: mongodb://localhost:27017)
 - `--mongodb-db`: MongoDB database name (default: geonames)
 - `--mongodb-collection`: MongoDB collection name (default: gazetteer)
 
-Available feature classes:
-- `A`: Country, state, region
-- `H`: Stream, lake
-- `L`: Parks, area
-- `P`: City, village
-- `R`: Road, railroad
-- `S`: Spot, building, farm
-- `T`: Mountain, hill, rock
-- `U`: Undersea
-- `V`: Forest, heath
-
 The Gazetteer data includes:
 - Geoname ID
 - Name (with ASCII and alternate names)
@@ -184,16 +182,14 @@ The MongoDB collection is indexed for efficient queries:
 
 #### MongoDB Format
 
-In MongoDB, the gazetteer data has the same structure as JSON but includes an additional `location` field for geospatial queries:
+In MongoDB, the gazetteer data uses slightly different field names and includes a `location` field for geospatial queries:
 
 ```json
 {
-    "geoname_id": 1609350,
+    "geonameid": 1609350,
     "name": "Bangkok",
-    "ascii_name": "Bangkok",
-    "alternate_names": ["Krung Thep", "กรุงเทพมหานคร"],
-    "latitude": 13.75,
-    "longitude": 100.51667,
+    "asciiname": "Bangkok",
+    "alternatenames": ["Krung Thep", "กรุงเทพมหานคร"],
     "location": {
         "type": "Point",
         "coordinates": [100.51667, 13.75]
@@ -217,11 +213,6 @@ In MongoDB, the gazetteer data has the same structure as JSON but includes an ad
 ```
 
 The MongoDB collection is indexed for efficient queries:
-- Unique index on `geoname_id`
-- Index on `country_code`
-- Index on `feature_class`
-- Index on `feature_code`
-- Text index on `name` and `ascii_name`
 - Geospatial index on `location`
 
 ## MongoDB Usage Examples
@@ -267,6 +258,26 @@ foreach ($result as $postalCode) {
 }
 ```
 
+## Error Handling
+
+The library throws `Farzai\Geonames\Exceptions\GeonamesException` for all error conditions:
+
+```php
+use Farzai\Geonames\Exceptions\GeonamesException;
+
+try {
+    // Download or convert operations
+} catch (GeonamesException $e) {
+    echo "Error: " . $e->getMessage();
+}
+```
+
+Common error scenarios:
+- **File operation failures**: Unable to read, write, or open files
+- **ZIP extraction failures**: Corrupted or invalid ZIP archives
+- **Data not found**: Missing expected data files in downloaded archives
+- **Missing dependencies**: MongoDB extension not installed when using MongoDB format
+
 ## License
 
 This package is open-sourced software licensed under the MIT license.

bin/geonames

@@ -5,11 +5,14 @@ declare(strict_types=1);
 
 require __DIR__ . '/../vendor/autoload.php';
 
-use Symfony\Component\Console\Application;
-use Farzai\Geonames\Console\Commands\DownloadPostalCodesCommand;
+use Composer\InstalledVersions;
 use Farzai\Geonames\Console\Commands\DownloadGazetteerCommand;
+use Farzai\Geonames\Console\Commands\DownloadPostalCodesCommand;
+use Symfony\Component\Console\Application;
+
+$version = InstalledVersions::getPrettyVersion('farzai/geonames') ?? '1.0.0';
 
-$application = new Application('Geonames CLI', '1.0.0');
+$application = new Application('Geonames CLI', $version);
 $application->add(new DownloadPostalCodesCommand());
 $application->add(new DownloadGazetteerCommand());
-$application->run(); 
+$application->run();

composer.json

@@ -7,15 +7,16 @@
     "require": {
         "php": "^8.1",
         "symfony/console": "^6.0 || ^7.0",
-        "guzzlehttp/guzzle": "^7.0"
+        "farzai/transport": "^2.1"
     },
     "require-dev": {
-        "pestphp/pest": "^2.34",
-        "spatie/ray": "^1.28",
         "laravel/pint": "^1.2",
+        "pestphp/pest": "^2.34",
         "phpstan/extension-installer": "^1.1",
         "phpstan/phpstan-deprecation-rules": "^1.0",
-        "phpstan/phpstan-phpunit": "^1.0"
+        "phpstan/phpstan-phpunit": "^1.0",
+        "spatie/ray": "^1.28",
+        "symfony/http-client": "^6.4 || ^7.0"
     },
     "autoload": {
         "psr-4": {
@@ -48,7 +49,6 @@
         "analyse": "vendor/bin/phpstan analyse"
     },
     "suggest": {
-        "guzzlehttp/guzzle": "Required to download the data from Geonames.",
         "ext-zip": "Required to extract the downloaded data.",
         "mongodb/mongodb": "Required for MongoDB output format support."
     },

src/Console/Commands/DownloadGazetteerCommand.php

@@ -7,6 +7,7 @@
 use Farzai\Geonames\Converter\GazetteerConverter;
 use Farzai\Geonames\Converter\MongoDBGazetteerConverter;
 use Farzai\Geonames\Downloader\GazetteerDownloader;
+use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -35,6 +36,10 @@
  *   geonames:gazetteer:download all -c P            # All populated places
  *   geonames:gazetteer:download US -f mongodb       # Import US data to MongoDB
  */
+#[AsCommand(
+    name: 'geonames:gazetteer:download',
+    description: 'Download and convert Geonames Gazetteer data'
+)]
 class DownloadGazetteerCommand extends Command
 {
     /**
@@ -45,20 +50,6 @@ class DownloadGazetteerCommand extends Command
         'admin2Codes.txt',
     ];
 
-    /**
-     * The default command name.
-     *
-     * @var string
-     */
-    protected static $defaultName = 'geonames:gazetteer:download';
-
-    /**
-     * The default command description.
-     *
-     * @var string
-     */
-    protected static $defaultDescription = 'Download and convert Geonames Gazetteer data';
-
     /**
      * The gazetteer downloader instance.
      */

src/Console/Commands/DownloadPostalCodesCommand.php

@@ -7,6 +7,7 @@
 use Farzai\Geonames\Converter\MongoDBPostalCodeConverter;
 use Farzai\Geonames\Converter\PostalCodeConverter;
 use Farzai\Geonames\Downloader\GeonamesDownloader;
+use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -24,22 +25,12 @@
  *   geonames:download all             # Download all countries
  *   geonames:download US -f mongodb   # Import US data to MongoDB
  */
+#[AsCommand(
+    name: 'geonames:download',
+    description: 'Download and convert postal codes data from Geonames'
+)]
 class DownloadPostalCodesCommand extends Command
 {
-    /**
-     * The default command name.
-     *
-     * @var string
-     */
-    protected static $defaultName = 'geonames:download';
-
-    /**
-     * The default command description.
-     *
-     * @var string
-     */
-    protected static $defaultDescription = 'Download and convert postal codes data from Geonames';
-
     /**
      * The postal code downloader instance.
      */

src/Converter/AbstractConverter.php

@@ -234,7 +234,7 @@ protected function streamPostalCodeRecords(string $txtFile): Generator
      */
     protected function createProgressBar(int $totalLines): ?ProgressBar
     {
-        if ($this->output === null) {
+        if ($this->output === null || $totalLines <= 0) {
             return null;
         }
 

src/Converter/GazetteerConverter.php

@@ -5,51 +5,113 @@
 namespace Farzai\Geonames\Converter;
 
 use Farzai\Geonames\Exceptions\GeonamesException;
+use Generator;
 
 /**
  * Converts GeoNames gazetteer data from ZIP files to JSON format.
  *
  * This converter extracts geographical feature data from GeoNames ZIP archives
  * and outputs it as a JSON file with administrative code name resolution.
+ * Uses streaming to handle large files with minimal memory usage.
  */
 class GazetteerConverter extends AbstractGazetteerConverter
 {
     /**
      * Process the gazetteer data file and write to JSON output.
      *
+     * Uses streaming to process large files with O(1) memory complexity.
+     *
      * @param  string  $txtFile  Path to the source TXT file containing gazetteer data
      * @param  string  $outputFile  Path to the output JSON file
      *
      * @throws GeonamesException When processing fails
      */
     protected function processFile(string $txtFile, string $outputFile): void
     {
-        $data = [];
-        $lines = file($txtFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
+        $totalLines = $this->countLines($txtFile);
+        $progressBar = $this->createProgressBar($totalLines);
 
-        if ($lines === false) {
-            throw GeonamesException::fileOperationFailed('read', $txtFile);
+        $handle = fopen($outputFile, 'wb');
+        if ($handle === false) {
+            throw GeonamesException::fileOperationFailed('open for writing', $outputFile);
         }
 
-        foreach ($lines as $line) {
-            if (empty(trim($line))) {
-                continue;
-            }
+        try {
+            $this->writeToHandle($handle, '[', $outputFile);
+            $first = true;
+
+            foreach ($this->streamGazetteerRecords($txtFile) as $record) {
+                if (! $first) {
+                    $this->writeToHandle($handle, ',', $outputFile);
+                }
+
+                $json = json_encode($record, JSON_UNESCAPED_UNICODE);
+                if ($json === false) {
+                    throw GeonamesException::fileOperationFailed('encode JSON', $outputFile);
+                }
 
-            $record = $this->parseGazetteerLine($line);
-            if ($record !== null) {
-                $data[] = $record;
+                $this->writeToHandle($handle, $json, $outputFile);
+                $first = false;
+
+                $progressBar?->advance();
             }
-        }
 
-        $jsonContent = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
-        if ($jsonContent === false) {
-            throw GeonamesException::fileOperationFailed('encode JSON', $outputFile);
+            $this->writeToHandle($handle, ']', $outputFile);
+        } finally {
+            fclose($handle);
+            $this->finishProgressBar($progressBar);
         }
+    }
 
-        $result = file_put_contents($outputFile, $jsonContent);
-        if ($result === false) {
+    /**
+     * Write content to a file handle with error checking.
+     *
+     * @param  resource  $handle  The file handle to write to
+     * @param  string  $content  The content to write
+     * @param  string  $outputFile  The output file path (for error messages)
+     *
+     * @throws GeonamesException When the write operation fails
+     */
+    private function writeToHandle($handle, string $content, string $outputFile): void
+    {
+        if (fwrite($handle, $content) === false) {
             throw GeonamesException::fileOperationFailed('write', $outputFile);
         }
     }
+
+    /**
+     * Stream gazetteer records from a TXT file.
+     *
+     * Uses a generator to yield records one at a time, enabling memory-efficient
+     * processing of large files.
+     *
+     * @param  string  $txtFile  Path to the TXT file containing gazetteer data
+     * @return Generator<int, array<string, mixed>> Yields gazetteer records
+     *
+     * @throws GeonamesException When the file cannot be opened
+     */
+    protected function streamGazetteerRecords(string $txtFile): Generator
+    {
+        $handle = fopen($txtFile, 'r');
+
+        if ($handle === false) {
+            throw GeonamesException::fileOperationFailed('open', $txtFile);
+        }
+
+        try {
+            while (($line = fgets($handle)) !== false) {
+                $trimmedLine = trim($line);
+                if (empty($trimmedLine)) {
+                    continue;
+                }
+
+                $record = $this->parseGazetteerLine($trimmedLine);
+                if ($record !== null) {
+                    yield $record;
+                }
+            }
+        } finally {
+            fclose($handle);
+        }
+    }
 }