Java 读取任意shapefile的所有字段，并插入到MongoDB数据库（Spring Boot）

---

Java 读取任意shapefile的所有字段，并插入到MongoDB数据库（Spring Boot）

• 亮点：动态获取字段信息，动态创建数据库表信息，支持任意shapefile数据
• pom

<repositories>
        <repository>
            <id>maven2-repository.dev.java.net</id>
            <name>Java.net repository</name>
            <url>http://download.java.net/maven/2</url>
        </repository>
        <repository>
            <id>osgeo</id>
            <name>Open Source Geospatial Foundation Repository</name>
            <url>http://download.osgeo.org/webdav/geotools/</url>
        </repository>
        <repository>
            <snapshots>
                <enabled>true</enabled>
            </snapshots>
            <id>boundless</id>
            <name>Boundless Maven Repository</name>
            <url>http://repo.boundlessgeo.com/main</url>
        </repository>
</repositories>

<dependency>
  <groupId>org.geotools</groupId>
    <artifactId>gt-shapefile</artifactId>
    <version>18.4</version>
</dependency>

<dependency>
    <groupId>org.geotools</groupId>
    <artifactId>gt-geojson</artifactId>
    <version>18.4</version>
</dependency>


<dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>mongodb-driver</artifactId>
    <version>3.6.2</version>
</dependency>

<dependency>
    <groupId>com.google.code.gson</groupId>
    <artifactId>gson</artifactId>
    <version>2.8.2</version>
</dependency>

1. 统一返回结果封装

/**
 * @program: tool
 * @description:
 * @author: Mr.superbeyone
 * @create: 2018-10-18 16:37
 **/
@Data
public class JsonResult<T> {
    private Integer code;
    private String msg;
    private T data;

    //使用 lombok @Data注解 或者  Get Set方法 ...
}

2. shp文件数据实体封装

/**
 * @program: tool
 * @description:
 * @author: Mr.superbeyone
 * @create: 2018-10-16 12:12
 **/
@Data
public class ShapeModel implements Serializable {
    private String id;
    private String type;
    private String geometry;
    private String properties;
    //使用 lombok @Data注解 或者  Get Set方法 ...
}

3. 核心代码

/**
  * 读取shp文件
  *
  * @param filePath   读取的文件路径
  * @param collection mongoDB collection
  * @return
  */
private JsonResult<Set<String>> readShapeFile(String filePath, MongoCollection<Document> collection) {
        JsonResult<Set<String>> result = new JsonResult<>();
        Set<String> set = new HashSet<>();
        File folder = new File(filePath);
        if (!folder.isDirectory()) {
            if (folder.toString().endsWith(".shp")) {
                try {
                    return getShapeFile(folder, collection);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            } else {
                System.out.println("选择的文件后缀名不是.shp");
                return null;
            }
        } else {
            File[] files = folder.listFiles();
            if (files.length <= 0) {
                System.out.println("目录文件为空");
                return null;
            }

            for (File file : files) {
                if (!file.toString().endsWith(".shp")) {
                    continue;
                }
                try {
                    result = getShapeFile(file, collection);
                    Set fields = result.getData();
                    set.addAll(fields);
                } catch (Exception e) {
                    e.printStackTrace();
                }

            }
            result.setData(set);
        }

        return result;
    }





private JsonResult<Set<String>> getShapeFile(File file, MongoCollection<Document> collection) {
        JsonResult<Set<String>> result = new JsonResult();
        HashSet<String> fields = new HashSet<>();
        int number = 0;
        int index = 0;
        try {
            ShapefileDataStore shapefileDataStore = new ShapefileDataStore(file.toURI().toURL());
            shapefileDataStore.setCharset(Charset.forName("utf-8"));
            ContentFeatureSource featureSource = shapefileDataStore.getFeatureSource();
            SimpleFeatureIterator featureIterator = featureSource.getFeatures().features();
            ArrayList<Document> documents = new ArrayList<>();
            Document document = null;
            while (featureIterator.hasNext()) {

                number++;
                SimpleFeature feature = featureIterator.next();
                Collection<? extends Property> properties = feature.getValue();
                //feature转GeoJSON
                FeatureJSON featureJSON = new FeatureJSON();
                StringWriter writer = new StringWriter();
                featureJSON.writeFeature(feature, writer);
				//TODO: 判断writer.toString().length() 否则可能会抛下列异常
				//org.bson.BsonSerializationException: Payload document size of is larger than maximum of 16777216

                ShapeModel shapeModel = JSON.parseObject(writer.toString(), ShapeModel.class);
                document = new Document();

                
                if (properties.size() < 0) {
                    continue;
                }
                if (index == 0) {
                    for (Property property : properties) {
                        if (property.getType().getClass() == org.geotools.feature.type.AttributeTypeImpl.class) {
                            fields.add(property.getName().toString());
                        } else {
                        	//空间字段
                            fields.add("#" + property.getName() + "#");
                        }
                    }
                    
                    result.setData(fields);
                    result.setCode(ResultCodeEnum.GET_UPDATE_FILE_FIELDS_SUCCESS.getCode());
                    result.setMsg(ResultCodeEnum.GET_UPDATE_FILE_FIELDS_SUCCESS.getMsg());

                    //TODO:创建分片数据库
                    //createShardDatabase(fields);
                }
                
                for (Property property : properties) {
                    if (property.getType().getClass() == org.geotools.feature.type.AttributeTypeImpl.class) {
                        if (property.getValue() == null) {
                            document.put(property.getName().toString(), "");
                        } else {
                            document.put(property.getName().toString(), property.getValue().toString());
                        }

                    } else {
                    	//空间字段
                        document.put(property.getName().toString(), shapeModel.getGeometry());
                    }
                }
                
                index++;
                //逐条插入
               //collection.insertOne(document);
                documents.add(document);

				//每1000条数据插入一次
                if (number % 1000 == 0) {
                    collection.insertMany(documents);
                    documents = new ArrayList<>();
                }
                //最后插入不足1000条的数据
                if (!featureIterator.hasNext()) {
                    collection.insertMany(documents);
                }

            }
        } catch (Exception e) {
            //出现异常
            result.setMsg("插入第" + (index + 1) + "条时出现错误");
            result.setCode(4000);            
        } finally {
        	featureIterator.close();
        }

        return result;
    }
}

3. 可能出现的异常

org.bson.BsonSerializationException: Payload document size of is larger than maximum of 16777216

3.1 异常产生原因

• 当前插入的数据大于16M，会抛出此异常

3.2 解决方案

• 判断所要插入的数据的大小，如果大于16M，可以选择跳过，或者停止插入

4 扩展

4.1 创建数据库分片

public void createShardDatabase(Set<String> fields) {

        //TODO:使用 databaseName 生成策略
        String databaseName = "super";
        //TODO:使用 collectionName 生成策略
        String collectionName = "beyone";


        List<String> set = new ArrayList<>();

        Map<String, Object> map = new HashMap<String, Object>();


        if (fields.size() < 0) {
            return;
        }
        for (String field : fields) {
        	//空间字段
            if (field.startsWith("#") && field.endsWith("#")) {
                String splitField = field.split("#")[1];
                map.put(splitField, "hashed");
            } else {
                set.add(field);
            }
        }
		//mongoDBConfig 是通过注入获得的
		/**
		 * @Autowired
		 * MongoDBConfig mongoDBConfig;
		 */
        MongoClient client = new MongoClient(mongoDBConfig.getHost(), mongoDBConfig.getPort());
        MongoDatabase adminDB = client.getDatabase(mongoDBConfig.getDatabase());
        Document enableShardDoc = new Document();
        enableShardDoc.put("enablesharding", databaseName);
        Document result = adminDB.runCommand(enableShardDoc);
        Object obj = result.get("ok");
        if (null == obj) {
        }
        if ("1.0".equals(obj.toString())) {
            Document shardKeyDoc = new Document();
            String coll = databaseName + "." + collectionName;
            shardKeyDoc.put("shardcollection", coll);
            Document t = new Document();
            t.putAll(map);
            shardKeyDoc.put("key", t);
            result = adminDB.runCommand(shardKeyDoc);
            obj = result.get("ok");
            if (null == obj) {

                //logger.info("error");
            }
            if ("1.0".equals(obj.toString())) {
                //logger.info("success");

            } else {
                //logger.info("error");
            }
        } else {
            //logger.info("error");
        }

    }