berkus
/
bincode-orig
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP Implement nocopy support for bincode (#150) 

* basic infrastructure

* use BincodeRead internally

* fix zero copy behavior

* rearrange module layout; remove pub(crate)

* add size-limit safety checks when deserializing str or bytes

* fix-up tests
This commit is contained in:
Ty Overby 2017-04-20 18:27:22 -07:00 committed by GitHub
parent 4654645c82
commit 6692674f43
7 changed files with 193 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.toml
View File

@ -14,8 +14,8 @@ description = "A binary serialization / deserialization strategy that uses Serde
[dependencies]
byteorder = "1.0.0"
num-traits = "0.1.32"
serde = { git = "https://github.com/serde-rs/serde", branch = "master" }
serde = "1.*.*"
[dev-dependencies]
serde_bytes = { git = "https://github.com/serde-rs/bytes", branch = "master" }
serde_derive = { git = "https://github.com/serde-rs/serde", branch = "master" }
serde_bytes = "0.10.*"
serde_derive = "1.*.*"

23
src/serde/reader.rs → src/de/mod.rs
View File

@ -8,6 +8,9 @@ use serde_crate::de::IntoDeserializer;
use serde_crate::de::Error as DeError;
use ::SizeLimit;
use super::{Result, Error, ErrorKind};
use self::read::BincodeRead;
pub mod read;
const BLOCK_SIZE: usize = 65536;
@ -30,7 +33,7 @@ pub struct Deserializer<R, S: SizeLimit, E: ByteOrder> {
_phantom: PhantomData<E>,
}
impl<R: Read, E: ByteOrder, S: SizeLimit> Deserializer<R, S, E> {
impl<'de, R: BincodeRead<'de>, E: ByteOrder, S: SizeLimit> Deserializer<R, S, E> {
/// Creates a new Deserializer with a given `Read`er and a size_limit.
pub fn new(r: R, size_limit: S) -> Deserializer<R, S, E> {
Deserializer {
@ -91,7 +94,7 @@ macro_rules! impl_nums {
}
impl<'de, 'a, R, S, E> serde::Deserializer<'de> for &'a mut Deserializer<R, S, E>
where R: Read, S: SizeLimit, E: ByteOrder {
where R: BincodeRead<'de>, S: SizeLimit, E: ByteOrder {
type Error = Error;
#[inline]
@ -181,7 +184,9 @@ where R: Read, S: SizeLimit, E: ByteOrder {
fn deserialize_str<V>(self, visitor: V) -> Result<V::Value>
where V: serde::de::Visitor<'de>,
{
visitor.visit_str(&try!(self.read_string()))
let len: usize = try!(serde::Deserialize::deserialize(&mut *self));
try!(self.read_bytes(len as u64));
self.reader.forward_read_str(len, visitor)
}
fn deserialize_string<V>(self, visitor: V) -> Result<V::Value>
@ -193,7 +198,9 @@ where R: Read, S: SizeLimit, E: ByteOrder {
fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value>
where V: serde::de::Visitor<'de>,
{
visitor.visit_bytes(&try!(self.read_vec()))
let len: usize = try!(serde::Deserialize::deserialize(&mut *self));
try!(self.read_bytes(len as u64));
self.reader.forward_read_bytes(len, visitor)
}
fn deserialize_byte_buf<V>(self, visitor: V) -> Result<V::Value>
@ -209,7 +216,7 @@ where R: Read, S: SizeLimit, E: ByteOrder {
where V: serde::de::Visitor<'de>,
{
impl<'de, 'a, R: 'a, S, E> serde::de::EnumAccess<'de> for &'a mut Deserializer<R, S, E>
where R: Read, S: SizeLimit, E: ByteOrder {
where R: BincodeRead<'de>, S: SizeLimit, E: ByteOrder {
type Error = Error;
type Variant = Self;
@ -235,7 +242,7 @@ where R: Read, S: SizeLimit, E: ByteOrder {
len: usize,
}
impl<'de, 'a, 'b: 'a, R: Read + 'b, S: SizeLimit, E: ByteOrder> serde::de::SeqAccess<'de> for Access<'a, R, S, E> {
impl<'de, 'a, 'b: 'a, R: BincodeRead<'de>+ 'b, S: SizeLimit, E: ByteOrder> serde::de::SeqAccess<'de> for Access<'a, R, S, E> {
type Error = Error;
fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>>
@ -288,7 +295,7 @@ where R: Read, S: SizeLimit, E: ByteOrder {
len: usize,
}
impl<'de, 'a, 'b: 'a, R: Read + 'b, S: SizeLimit, E: ByteOrder> serde::de::MapAccess<'de> for Access<'a, R, S, E> {
impl<'de, 'a, 'b: 'a, R: BincodeRead<'de> + 'b, S: SizeLimit, E: ByteOrder> serde::de::MapAccess<'de> for Access<'a, R, S, E> {
type Error = Error;
fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>>
@ -372,7 +379,7 @@ where R: Read, S: SizeLimit, E: ByteOrder {
}
impl<'de, 'a, R, S, E> serde::de::VariantAccess<'de> for &'a mut Deserializer<R, S, E>
where R: Read, S: SizeLimit, E: ByteOrder {
where R: BincodeRead<'de>, S: SizeLimit, E: ByteOrder {
type Error = Error;
fn unit_variant(self) -> Result<()> {

120
src/de/read.rs Normal file
View File

@ -0,0 +1,120 @@
use std::io::{Read as IoRead, Result as IoResult, Error as IoError, ErrorKind as IoErrorKind};
use ::Result;
use serde_crate as serde;
/// A byte-oriented reading trait that is specialized for
/// slices and generic readers.
pub trait BincodeRead<'storage>: IoRead {
#[doc(hidden)]
fn forward_read_str<V>(&mut self, length: usize, visitor: V) -> Result<V::Value>
where V: serde::de::Visitor<'storage>;
#[doc(hidden)]
fn forward_read_bytes<V: serde::de::Visitor<'storage>>(&mut self, length: usize, visitor: V) -> Result<V::Value>
where V: serde::de::Visitor<'storage>;
}
/// A BincodeRead implementation for byte slices
pub struct SliceReader<'storage> {
slice: &'storage [u8]
}
/// A BincodeRead implementation for io::Readers
pub struct IoReadReader<R> {
reader: R,
temp_buffer: Vec<u8>,
}
impl <'storage> SliceReader<'storage> {
/// Constructs a slice reader
pub fn new(bytes: &'storage [u8]) -> SliceReader<'storage> {
SliceReader {
slice: bytes,
}
}
}
impl <R> IoReadReader<R> {
/// Constructs an IoReadReader
pub fn new(r: R) -> IoReadReader<R> {
IoReadReader {
reader: r,
temp_buffer: vec![],
}
}
}
impl <'storage> IoRead for SliceReader<'storage> {
fn read(&mut self, out: & mut [u8]) -> IoResult<usize> {
(&mut self.slice).read(out)
}
}
impl <R: IoRead> IoRead for IoReadReader<R> {
fn read(&mut self, out: & mut [u8]) -> IoResult<usize> {
self.reader.read(out)
}
}
impl <'storage> BincodeRead<'storage> for SliceReader<'storage> {
fn forward_read_str<V: serde::de::Visitor<'storage>>(&mut self, length: usize, visitor: V) -> Result<V::Value> {
use ::ErrorKind;
if length > self.slice.len() {
return Err(Box::new(ErrorKind::IoError(IoError::new(IoErrorKind::UnexpectedEof, ""))));
}
let string = match ::std::str::from_utf8(&self.slice[..length]) {
Ok(s) => s,
Err(_) => return Err(Box::new(ErrorKind::InvalidEncoding {
desc: "string was not valid utf8",
detail: None,
})),
};
let r = visitor.visit_borrowed_str(string);
self.slice = &self.slice[length..];
r
}
fn forward_read_bytes<V: serde::de::Visitor<'storage>>(&mut self, length: usize, visitor: V) -> Result<V::Value> {
use ::ErrorKind;
if length > self.slice.len() {
return Err(Box::new(ErrorKind::IoError(IoError::new(IoErrorKind::UnexpectedEof, ""))));
}
let r = visitor.visit_borrowed_bytes(&self.slice[..length]);
self.slice = &self.slice[length..];
r
}
}
impl <R> BincodeRead<'static> for IoReadReader<R> where R: IoRead {
fn forward_read_str<V: serde::de::Visitor<'static>>(&mut self, length: usize, visitor: V) -> Result<V::Value> {
use ::ErrorKind;
let current_length = self.temp_buffer.len();
if length > current_length{
self.temp_buffer.reserve_exact(length - current_length);
}
self.reader.read_exact(&mut self.temp_buffer[..length])?;
let string = match ::std::str::from_utf8(&self.temp_buffer[..length]) {
Ok(s) => s,
Err(_) => return Err(Box::new(ErrorKind::InvalidEncoding {
desc: "string was not valid utf8",
detail: None,
})),
};
let r = visitor.visit_str(string);
r
}
fn forward_read_bytes<V: serde::de::Visitor<'static>>(&mut self, length: usize, visitor: V) -> Result<V::Value> {
let current_length = self.temp_buffer.len();
if length > current_length{
self.temp_buffer.reserve_exact(length - current_length);
}
self.reader.read_exact(&mut self.temp_buffer[..length])?;
let r = visitor.visit_bytes(&self.temp_buffer[..length]);
r
}
}

19
src/serde/mod.rs → src/internal.rs
View File

@ -8,21 +8,18 @@ use std::{error, fmt, result};
use ::SizeLimit;
use byteorder::{ByteOrder};
pub use self::reader::{
pub use super::de::{
Deserializer,
};
pub use self::writer::{
pub use super::ser::{
Serializer,
};
use self::writer::SizeChecker;
use super::ser::SizeChecker;
use serde_crate as serde;
mod reader;
mod writer;
/// The result of a serialization or deserialization operation.
pub type Result<T> = result::Result<T, Error>;
@ -225,6 +222,7 @@ pub fn serialized_size_bounded<T: ?Sized>(value: &T, max: u64) -> Option<u64>
pub fn deserialize_from<R: ?Sized, T, S, E>(reader: &mut R, size_limit: S) -> Result<T>
where R: Read, T: serde::de::DeserializeOwned, S: SizeLimit, E: ByteOrder
{
let reader = ::de::read::IoReadReader::new(reader);
let mut deserializer = Deserializer::<_, S, E>::new(reader, size_limit);
serde::Deserialize::deserialize(&mut deserializer)
}
@ -233,9 +231,10 @@ pub fn deserialize_from<R: ?Sized, T, S, E>(reader: &mut R, size_limit: S) -> Re
///
/// This method does not have a size-limit because if you already have the bytes
/// in memory, then you don't gain anything by having a limiter.
pub fn deserialize<T, E: ByteOrder>(bytes: &[u8]) -> Result<T>
where T: serde::de::DeserializeOwned,
pub fn deserialize<'a, T, E: ByteOrder>(bytes: &'a [u8]) -> Result<T>
where T: serde::de::Deserialize<'a>,
{
let mut reader = bytes;
deserialize_from::<_, _, _, E>(&mut reader, super::Infinite)
let reader = ::de::read::SliceReader::new(bytes);
let mut deserializer = Deserializer::<_, _, E>::new(reader, super::Infinite);
serde::Deserialize::deserialize(&mut deserializer)
}

34
src/lib.rs
View File

@ -40,30 +40,32 @@ extern crate byteorder;
extern crate num_traits;
extern crate serde as serde_crate;
mod serde;
mod ser;
mod de;
pub mod internal;
/// All of the core bincode functions and types with the ability to choose endianness.
pub mod endian_choice {
pub use super::serde::{Deserializer, Serializer, serialize, serialize_into, deserialize, deserialize_from};
pub mod read_types {
//! The types that the deserializer uses for optimizations
pub use ::de::read::{SliceReader, BincodeRead, IoReadReader};
}
use std::io::{Read, Write};
pub use serde::{ErrorKind, Error, Result, serialized_size, serialized_size_bounded};
pub use internal::{ErrorKind, Error, Result, serialized_size, serialized_size_bounded};
/// A Deserializer that uses LittleEndian byteorder
pub type Deserializer<W, S> = serde::Deserializer<W, S, byteorder::LittleEndian>;
pub type Deserializer<W, S> = internal::Deserializer<W, S, byteorder::LittleEndian>;
/// A Serializer that uses LittleEndian byteorder
pub type Serializer<W> = serde::Serializer<W, byteorder::LittleEndian>;
pub type Serializer<W> = internal::Serializer<W, byteorder::LittleEndian>;
/// Deserializes a slice of bytes into an object.
///
/// This method does not have a size-limit because if you already have the bytes
/// in memory, then you don't gain anything by having a limiter.
pub fn deserialize<T>(bytes: &[u8]) -> serde::Result<T>
where T: serde_crate::de::DeserializeOwned,
pub fn deserialize<'a, T>(bytes: &'a [u8]) -> internal::Result<T>
where T: serde_crate::de::Deserialize<'a>,
{
serde::deserialize::<_, byteorder::LittleEndian>(bytes)
internal::deserialize::<_, byteorder::LittleEndian>(bytes)
}
/// Deserializes an object directly from a `Buffer`ed Reader.
@ -75,10 +77,10 @@ pub fn deserialize<T>(bytes: &[u8]) -> serde::Result<T>
/// If this returns an `Error`, assume that the buffer that you passed
/// in is in an invalid state, as the error could be returned during any point
/// in the reading.
pub fn deserialize_from<R: ?Sized, T, S>(reader: &mut R, size_limit: S) -> serde::Result<T>
pub fn deserialize_from<R: ?Sized, T, S>(reader: &mut R, size_limit: S) -> internal::Result<T>
where R: Read, T: serde_crate::de::DeserializeOwned, S: SizeLimit
{
serde::deserialize_from::<_, _, _, byteorder::LittleEndian>(reader, size_limit)
internal::deserialize_from::<_, _, _, byteorder::LittleEndian>(reader, size_limit)
}
/// Serializes an object directly into a `Writer`.
@ -89,20 +91,20 @@ pub fn deserialize_from<R: ?Sized, T, S>(reader: &mut R, size_limit: S) -> serde
/// If this returns an `Error` (other than SizeLimit), assume that the
/// writer is in an invalid state, as writing could bail out in the middle of
/// serializing.
pub fn serialize_into<W: ?Sized, T: ?Sized, S>(writer: &mut W, value: &T, size_limit: S) -> serde::Result<()>
pub fn serialize_into<W: ?Sized, T: ?Sized, S>(writer: &mut W, value: &T, size_limit: S) -> internal::Result<()>
where W: Write, T: serde_crate::Serialize, S: SizeLimit
{
serde::serialize_into::<_, _, _, byteorder::LittleEndian>(writer, value, size_limit)
internal::serialize_into::<_, _, _, byteorder::LittleEndian>(writer, value, size_limit)
}
/// Serializes a serializable object into a `Vec` of bytes.
///
/// If the serialization would take more bytes than allowed by `size_limit`,
/// an error is returned.
pub fn serialize<T: ?Sized, S>(value: &T, size_limit: S) -> serde::Result<Vec<u8>>
pub fn serialize<T: ?Sized, S>(value: &T, size_limit: S) -> internal::Result<Vec<u8>>
where T: serde_crate::Serialize, S: SizeLimit
{
serde::serialize::<_, _, byteorder::LittleEndian>(value, size_limit)
internal::serialize::<_, _, byteorder::LittleEndian>(value, size_limit)
}
/// A limit on the amount of bytes that can be read or written.

2
src/serde/writer.rs → src/ser/mod.rs
View File

@ -7,7 +7,7 @@ use serde_crate as serde;
use byteorder::{WriteBytesExt, ByteOrder};
use super::{Result, Error, ErrorKind};
use super::super::SizeLimit;
use super::SizeLimit;
/// An Serializer that encodes values directly into a Writer.
///

36
tests/test.rs
View File

@ -12,7 +12,7 @@ use std::borrow::Cow;
use bincode::{Infinite, Bounded};
use bincode::{serialized_size, ErrorKind, Result};
use bincode::endian_choice::{serialize, deserialize};
use bincode::internal::{serialize, deserialize, deserialize_from};
use bincode::serialize as serialize_little;
use bincode::deserialize as deserialize_little;
@ -22,23 +22,22 @@ fn the_same<V>(element: V)
where V: serde::Serialize+serde::de::DeserializeOwned+PartialEq+Debug+'static
{
let size = serialized_size(&element);
{
let encoded = serialize_little(&element, Infinite);
let encoded = encoded.unwrap();
let decoded = deserialize_little(&encoded[..]);
let decoded = decoded.unwrap();
let encoded = serialize_little(&element, Infinite).unwrap();
let decoded = deserialize_little(&encoded[..]).unwrap();
assert_eq!(element, decoded);
assert_eq!(size, encoded.len() as u64);
}
{
let encoded = serialize::<_, _, byteorder::BigEndian>(&element, Infinite);
let encoded = encoded.unwrap();
let decoded = deserialize::<_, byteorder::BigEndian>(&encoded[..]);
let decoded = decoded.unwrap();
let encoded = serialize::<_, _, byteorder::BigEndian>(&element, Infinite).unwrap();
let decoded = deserialize::<_, byteorder::BigEndian>(&encoded[..]).unwrap();
let decoded_reader = deserialize_from::<_, _, _, byteorder::BigEndian>(&mut &encoded[..], Infinite).unwrap();
assert_eq!(element, decoded);
assert_eq!(element, decoded_reader);
assert_eq!(size, encoded.len() as u64);
}
}
@ -399,3 +398,22 @@ fn endian_difference() {
let big = serialize::<_, _, byteorder::BigEndian>(&x, Infinite).unwrap();
assert_ne!(little, big);
}
#[test]
fn test_zero_copy_parse() {
#[derive(Serialize, Deserialize, Eq, PartialEq, Debug)]
struct Foo<'a> {
borrowed_str: &'a str,
borrowed_bytes: &'a [u8],
}
let f = Foo {
borrowed_str: "hi",
borrowed_bytes: &[0, 1, 2, 3],
};
{
let encoded = serialize_little(&f, Infinite).unwrap();
let out: Foo = deserialize_little(&encoded[..]).unwrap();
assert_eq!(out, f);
}
}