使用本文和Json.Net从文件中存储和检索大型Json(100 mb +)的有效方法
我一直试图找出使用JSon.net从包含Json的文件存储和检索数据的最佳方法。 在我的研究期间,我找到了几种方法。
- http://www.drdobbs.com/windows/parsing-big-records-with-jsonnet/240165316
- http://www.ngdata.com/parsing-a-large-json-file-efficiently-and-easily/ Jackson Api(仅适用于Java)
任何这种方法都可以与Json.Net一起使用吗? DrDoobs的文章是否有可用的实际实现? 我无法弄清楚HandleToken(reader.TokenType,reader.Value)方法。
目前的统计数据:
-
大约5分钟写出大小约为60Mb的Json
-
大约3分钟阅读Json。
现行代码:
public static T DeserializeJsonFromStream(Stream s) { using (StreamReader reader = new StreamReader(s)) { using (JsonTextReader jsonReader = new JsonTextReader(reader)) { JsonSerializer ser = new JsonSerializer(); ser.Formatting = Newtonsoft.Json.Formatting.None; ser.PreserveReferencesHandling = PreserveReferencesHandling.Objects; ser.TypeNameHandling = TypeNameHandling.All; ser.NullValueHandling = NullValueHandling.Ignore; ser.Error += ReportJsonErrors; ser.DateFormatHandling = DateFormatHandling.IsoDateFormat; return ser.Deserialize(jsonReader); } } } public static void SerializeJsonIntoStream(object value, Stream s) { using (StreamWriter writer = new StreamWriter(s)) { using (JsonTextWriter jsonWriter = new JsonTextWriter(writer)) { JsonSerializer ser = new JsonSerializer(); ser.Formatting = Newtonsoft.Json.Formatting.None; ser.PreserveReferencesHandling = PreserveReferencesHandling.Objects; ser.TypeNameHandling = TypeNameHandling.All; ser.NullValueHandling = NullValueHandling.Ignore; ser.Error += ReportJsonErrors; ser.Serialize(jsonWriter, value); jsonWriter.Flush(); } } }
由于我不需要手动读取序列化文件,使用协议缓冲区解决了问题,序列化/反序列化相同数量的数据需要6秒。
它可以在Nuget 网站http://www.nuget.org/packages/protobuf-net上找到
public static async Task DeserializeFromBinary (string filename, StorageFolder storageFolder) { try { var file = await storageFolder.GetFileAsync(filename); if (file != null) { var stream = await file.OpenStreamForReadAsync(); T content = Serializer.Deserialize (stream); return content; } } catch (NullReferenceException nullException) { logger.LogError("Exception happened while de-serializing input object, Error: " + nullException.Message); } catch (FileNotFoundException fileNotFound) { logger.LogError("Exception happened while de-serializing input object, Error: " + fileNotFound.Message); } catch (Exception e) { logger.LogError("Exception happened while de-serializing input object, Error: " + e.Message, e.ToString()); } return default(T); } public static async Task SerializeIntoBinary(string fileName, StorageFolder destinationFolder, Content content) { bool shouldRetry = true; while (shouldRetry) { try { StorageFile file = await destinationFolder.CreateFileAsync(fileName, CreationCollisionOption.ReplaceExisting); using (var stream = await file.OpenStreamForWriteAsync()) { Serializer.Serialize(stream, content); shouldRetry = false; return true; } } catch (UnauthorizedAccessException unAuthorizedAccess) { logger.LogError("UnauthorizedAccessException happened while serializing input object, will wait for 5 seconds, Error: " + unAuthorizedAccess.Message); shouldRetry = true; } catch (NullReferenceException nullException) { shouldRetry = false; logger.LogError("Exception happened while serializing input object, Error: " + nullException.Message); } catch (Exception e) { shouldRetry = false; logger.LogError("Exception happened while serializing input object, Error: " + e.Message, e.ToString()); } if (shouldRetry) await Task.Delay(5000); } return false; }