README
Geojson Feature Reader
Reads large GeoJSON files and returns valid features one by one. Useful when read time does not matter.
Built to provide a solution to json streams backpressure overflowing when processing geometries before writing
NB blocks event loop incrementally
The callback emits a feature when found. The promise is resolved when alldata is read, returns the feature count.
Rate Average
30mb file - 1000 features per second using +-150mb memory for simple polygons.
- avg 1 min to process 30k features.
90mb file - 200-300 features per second using +-200mb memory for larger complex polygons.
- avg 10min to process 120k features.
300mb file > 20-50 features per second using +-250mb memory for larger complex polygons.
- avg 2hours to process 400k features
Read speed increases as the process runs
Install
npm install geojson-feature-reader
Usage
Base Example
const read_stream = fs.createReadStream;
const geojsonString = "{...}"
const feature_reader = require('geojson-feature-reader')
const options = {}
//with custom options
feature_reader(read_stream || geojsonString, options, feature_callback)
.then(count => console.log(`${count} features processed`))
.catch(e => console.error(`Unable to read data`))
//with default options
feature_reader(read_stream || geojsonString, feature_callback)
.then(count => console.log(`${count} features processed`))
.catch(e => console.error(`Unable to read data`))
//feature callback
function feature_callback(feature){
feature // string
}
Options
let options = {
// number
// The amount of features to process before waiting.
// Less features opens the event loop more often.
pause_feature_counter : 1000,
// number or function
// A. the wait time after the features counter is reached.
// B. a function to indicate when more features can be read.
// > opens event loop
pause_feature_timeout : 100,
// number
// Expand geometrycollections to individual features.
// Counts towards `pause_feature_counter`
expand_geom_collection : true
}
Basic Usage
let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson')
feature_reader(read_stream, feature_callback)
.then(count => console.log(`${count} features processed`))
.catch(e => console.error(`Unable to read data`))
function feature_callback(feature){
typeof feature === 'string' //true
console.log(feature)
// { "type": "Feature", "properties": { ... }, "geometry": { ... }
// or
// { "type": "feature", "properties": { ... }, "geometry": { ...} }
}
With options
let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson')
let options = {
pause_feature_counter : 1000,
pause_feature_timeout : 10,
expand_geom_collection : false
}
feature_reader(read_stream, options, feature_callback)
.then(count => console.log(`${count} features processed`))
.catch(e => console.error(`Unable to read data`))
function feature_callback(feature){
typeof feature === 'string' //true
console.log(feature)
// { "type": "Feature", "properties": { ... }, "geometry": { ... }
// or
// { "type": "feature", "properties": { ... }, "geometry": { ...} }
}
Handle readiness
let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson')
let options = {
pause_feature_counter : 1000,
pause_feature_timeout : continue_read, //wait until function returns true
expand_geom_collection : true
}
feature_reader(read_stream, feature_callback)
.then(count => console.log(`${count} features processed`))
.catch(e => console.error(`Unable to read data`))
function feature_callback(feature){
typeof feature === 'string' //true
console.log(feature)
// { "type": "Feature", "properties": { ... }, "geometry": { ... }
// or
// { "type": "feature", "properties": { ... }, "geometry": { ...} }
}
let ready_for_next = false;
function continue_read(){
// file reading resumes
// function is tested every 100ms once reading is pasued
return ready_for_next
}
//
setTimeout(() =>{
ready_for_next = true
},1000)
Provide complete featurecollection as string
let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson','utf8')
let allData = ''
read_stream
.on('data', data => allData += data)
.on('error', error => console.error('read fail'))
.on('end', () => extract_features(allData))
let options = {
pause_feature_counter : 1000,
pause_feature_timeout : 10,
expand_geom_collection : true
}
function extract_features(data){
feature_reader(data, feature_callback)
.then(count => console.log(`${count} features processed`))
.catch(e => console.error(`Unable to read data`))
}
function feature_callback(feature){
typeof feature === 'string' //true
console.log(feature)
// { "type": "Feature", "properties": { ... }, "geometry": { ... }
// or
// { "type": "feature", "properties": { ... }, "geometry": { ...} }
}
Dependencies
- none
Built With
- Node ReadStream - Native Node file reader
Authors
- startkev - gitLab
License
This project is licensed under the MIT License