Posted In: GSON, JSON

GSON TypeAdapter – How to process large JSON file

I have successfully tried this code with 90K records.

This is useful in following scenarios

When it is impossible or undesirable to load the entire object model into memory. This is most relevant on mobile platforms where memory is limited.
When it is necessary to read or write a document before it is completely available.

Sample JSON used

[
{"auc":234421082,"item":3821,"owner":"???","ownerRealm":"???",
"bid":84500,"buyout":90000,"quantity":1,"timeLeft":"VERY_LONG","rand":0,"seed":385088896},
{"auc":234242934,"item":72093,"owner":"???","ownerRealm":"???",
"bid":399999,"buyout":399999,"quantity":20,"timeLeft":"VERY_LONG","rand":0,"seed":517959808}

.....

Steps

1. Create POJO to fit JSON
2. Create adapter to parse required fields
3. Use GsonBuilder to register TypeAdapter

POJO
public class Auction {
	private int auc;
	private int item;
	private String owner;
	private int bid;

	public int getAuc() {
		return auc;
	}

........
com.google.gson.TypeAdapter
package com.javausecase.gson;

import java.io.IOException;

import com.google.gson.TypeAdapter;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;

public class AuctionAdapter extends TypeAdapter<Auction> {
	@Override
	public Auction read(final JsonReader in) throws IOException {
		final Auction auction = new Auction();
		while (in.hasNext()) {
			String field = in.nextName();
			switch (field) {
			case "auc":
				auction.setAuc(in.nextInt());
				break;
			case "item":
				auction.setItem(in.nextInt());
				break;
			case "owner":
				auction.setOwner(in.nextString());
				break;
			case "ownerRealm":
				in.nextString();
				break;
			case "bid":
				auction.setBid(in.nextInt());
				break;
			case "buyout":
				in.nextInt();
				break;
			case "quantity":
				in.nextInt();
				break;
			case "timeLeft":
				in.nextString();
				break;
			case "rand":
				in.nextInt();
				break;
			case "seed":
				in.nextLong();
				break;
			}
		}
		return auction;
	}

	@Override
	public void write(final JsonWriter out, final Auction auction)
			throws IOException {
		out.beginObject();
		out.name("auc").value(auction.getAuc());
		out.name("item").value(auction.getItem());
		out.name("owner").value(auction.getOwner());
		out.name("bid").value(auction.getBid());
		out.endObject();
	}
}
GsonBuilder
package com.javausecase.gson;

import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;

import org.junit.Test;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.stream.JsonReader;

public class ReadBigJson {
	@Test
	public void main() throws Exception {
		System.out.println("Start");
		InputStream in = new FileInputStream(
				"D:/abhijit/javausecase/auctions_bigjson.json");
		JsonReader reader = new JsonReader(new InputStreamReader(in, "UTF-8"));
		List<Auction> auctions = new ArrayList<Auction>();
		Gson gson = new GsonBuilder()
				.registerTypeAdapter(Auction.class, new AuctionAdapter())
				.create();
		reader.beginArray();
		while (reader.hasNext()) {
			reader.beginObject();
			Auction auction = gson.fromJson(reader, Auction.class);
			auctions.add(auction);
			reader.endObject();
		}
		reader.endArray();
		reader.close();
		System.out.println(auctions.size());
		System.out.println("End");
	}

}

Maven

<dependency>
    <groupId>com.google.code.gson</groupId>
    <artifactId>gson</artifactId>
    <version>2.8.2</version>
</dependency>

Reference

https://sites.google.com/site/gson/streaming

Tags:

by , on February 21st, 2018

  • Categories