From 87f5f9d4eb8dc41cb9173a7bdf01ab095a341fdb Mon Sep 17 00:00:00 2001 From: "Coren[m]" Date: Fri, 13 Sep 2013 18:05:11 +0200 Subject: State loading/saving: Instead of a blob, segment into sections marked with tags. Primary rationale: The part that DHT saves changes if IP is expanded to IPv6. To let people keep their friends/name, change the datafile format now, while everybody is still on the same page. Loading/Saving rewritten to allow a part of the file to be incomprehensible to the loading routine. Added a magic cookie at the beginning to mark the file as tox's. Changes in some part of the datafile can be skipped and the remaining parts still be consumed. Allows a wide margin of forward compatibility (like the IP to IPv6 transition, but also e.g. a change in the key format). As long as the file is not completely garbled, the routine will read as much as possible. Only the KEY section is considered mandatory: a malformed key section leads to a negative result. util.*: - holds the driving function which jumps from section to section and calls back with section length and tag (type) Messenger.c,DHT.*: - new loading functions call the util-function with a callback, which subsequently consumes the sections - old routines are kept to fall back onto if the magic cookie at the beginning isn't present - saving is still done in one local routine --- toxcore/DHT.c | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 3 deletions(-) (limited to 'toxcore/DHT.c') diff --git a/toxcore/DHT.c b/toxcore/DHT.c index e2d91256..774dd6f2 100644 --- a/toxcore/DHT.c +++ b/toxcore/DHT.c @@ -1236,13 +1236,13 @@ void kill_DHT(DHT *dht) } /* Get the size of the DHT (for saving). */ -uint32_t DHT_size(DHT *dht) +uint32_t DHT_size_old(DHT *dht) { return sizeof(dht->close_clientlist) + sizeof(DHT_Friend) * dht->num_friends; } /* Save the DHT in data where data is an array of size DHT_size(). */ -void DHT_save(DHT *dht, uint8_t *data) +void DHT_save_old(DHT *dht, uint8_t *data) { memcpy(data, dht->close_clientlist, sizeof(dht->close_clientlist)); memcpy(data + sizeof(dht->close_clientlist), dht->friends_list, sizeof(DHT_Friend) * dht->num_friends); @@ -1253,7 +1253,7 @@ void DHT_save(DHT *dht, uint8_t *data) * return -1 if failure. * return 0 if success. */ -int DHT_load(DHT *dht, uint8_t *data, uint32_t size) +int DHT_load_old(DHT *dht, uint8_t *data, uint32_t size) { if (size < sizeof(dht->close_clientlist)) return -1; @@ -1295,6 +1295,132 @@ int DHT_load(DHT *dht, uint8_t *data, uint32_t size) return 0; } + +/* new DHT format for load/save, more robust and forward compatible */ + +#define DHT_STATE_COOKIE_GLOBAL 0x159000d + +#define DHT_STATE_COOKIE_TYPE 0x11ce +#define DHT_STATE_TYPE_FRIENDS 1 +#define DHT_STATE_TYPE_CLIENTS 2 + +typedef uint16_t statelensub_t; + +/* Get the size of the DHT (for saving). */ +uint32_t DHT_size(DHT *dht) +{ + uint32_t num = 0, i; + for (i = 0; i < LCLIENT_LIST; ++i) + if (dht->close_clientlist[i].timestamp != 0) + num++; + + uint32_t size32 = sizeof(uint32_t), lengthsublen = sizeof(statelensub_t); + uint32_t sizesubhead = lengthsublen + size32; + return size32 + + sizesubhead + sizeof(DHT_Friend) * dht->num_friends + + sizesubhead + sizeof(Client_data) * num; +} + +static uint8_t *z_state_save_subheader(uint8_t *data, statelensub_t len, uint16_t type) +{ + *(statelensub_t *)data = len; + data += sizeof(statelensub_t); + *(uint32_t *)data = (DHT_STATE_COOKIE_TYPE << 16) | type; + data += sizeof(uint32_t); + return data; +} + +/* Save the DHT in data where data is an array of size DHT_size(). */ +void DHT_save(DHT *dht, uint8_t *data) +{ + statelensub_t len; + uint16_t type; + *(uint32_t *)data = DHT_STATE_COOKIE_GLOBAL; + data += sizeof(uint32_t); + + len = sizeof(DHT_Friend) * dht->num_friends; + type = DHT_STATE_TYPE_FRIENDS; + data = z_state_save_subheader(data, len, type); + memcpy(data, dht->friends_list, len); + data += len; + + uint32_t num = 0, i; + for (i = 0; i < LCLIENT_LIST; ++i) + if (dht->close_clientlist[i].timestamp != 0) + num++; + + if (!num) + return; + + len = num * sizeof(Client_data); + type = DHT_STATE_TYPE_CLIENTS; + data = z_state_save_subheader(data, len, type); + Client_data *clients = (Client_data *)data; + for (num = 0, i = 0; i < LCLIENT_LIST; ++i) + if (dht->close_clientlist[i].timestamp != 0) + memcpy(&clients[num++], &dht->close_clientlist[i], sizeof(Client_data)); + data += len; +} + +static int dht_load_state_callback(void *outer, uint8_t *data, uint32_t length, uint16_t type) +{ + DHT *dht = outer; + uint32_t num, i, j; + switch(type) { + case DHT_STATE_TYPE_FRIENDS: + if (length % sizeof(DHT_Friend) != 0) + break; + + DHT_Friend *friend_list = (DHT_Friend *)data; + num = length / sizeof(DHT_Friend); + for (i = 0; i < num; ++i) { + DHT_addfriend(dht, friend_list[i].client_id); + for (j = 0; j < MAX_FRIEND_CLIENTS; ++j) { + Client_data *client = &friend_list[i].client_list[j]; + if (client->timestamp != 0) + getnodes(dht, client->ip_port, client->client_id, friend_list[i].client_id); + } + } + + break; + + case DHT_STATE_TYPE_CLIENTS: + if ((length % sizeof(Client_data)) != 0) + break; + + num = length / sizeof(Client_data); + Client_data *client_list = (Client_data *)data; + for (i = 0; i < num; ++i) + if (client_list[i].timestamp != 0) + DHT_bootstrap(dht, client_list[i].ip_port, client_list[i].client_id); + + break; + + default: + fprintf(stderr, "Load state (DHT): contains unrecognized part (len %u, type %u)\n", + length, type); + } + + return 0; +} + +/* Load the DHT from data of size size. + * + * return -1 if failure. + * return 0 if success. + */ +int DHT_load_new(DHT *dht, uint8_t *data, uint32_t length) +{ + uint32_t cookie_len = sizeof(uint32_t); + if (length > cookie_len) { + uint32_t *data32 = (uint32_t *)data; + if (data32[0] == DHT_STATE_COOKIE_GLOBAL) + return load_state(dht_load_state_callback, dht, data + cookie_len, + length - cookie_len, DHT_STATE_COOKIE_TYPE); + } + + return DHT_load_old(dht, data, length); +} /* return 0 if we are not connected to the DHT. * return 1 if we are. */ -- cgit v1.2.3 From a341b259b6342962e209f1b50708abe40f1cdad2 Mon Sep 17 00:00:00 2001 From: "Coren[m]" Date: Sat, 14 Sep 2013 10:43:09 +0200 Subject: Change sublength to 32 bits. Someone might have more than (sizeof(Friends) / 65536) friends... --- toxcore/DHT.c | 17 +++++++---------- toxcore/Messenger.c | 35 +++++++++++++++-------------------- toxcore/util.c | 10 +++++----- toxcore/util.h | 1 - 4 files changed, 27 insertions(+), 36 deletions(-) (limited to 'toxcore/DHT.c') diff --git a/toxcore/DHT.c b/toxcore/DHT.c index 0b866940..fcd15686 100644 --- a/toxcore/DHT.c +++ b/toxcore/DHT.c @@ -1304,8 +1304,6 @@ int DHT_load_old(DHT *dht, uint8_t *data, uint32_t size) #define DHT_STATE_TYPE_FRIENDS 1 #define DHT_STATE_TYPE_CLIENTS 2 -typedef uint16_t statelensub_t; - /* Get the size of the DHT (for saving). */ uint32_t DHT_size(DHT *dht) { @@ -1314,26 +1312,25 @@ uint32_t DHT_size(DHT *dht) if (dht->close_clientlist[i].timestamp != 0) num++; - uint32_t size32 = sizeof(uint32_t), lengthsublen = sizeof(statelensub_t); - uint32_t sizesubhead = lengthsublen + size32; + uint32_t size32 = sizeof(uint32_t), sizesubhead = size32 * 2; return size32 + sizesubhead + sizeof(DHT_Friend) * dht->num_friends + sizesubhead + sizeof(Client_data) * num; } -static uint8_t *z_state_save_subheader(uint8_t *data, statelensub_t len, uint16_t type) +static uint8_t *z_state_save_subheader(uint8_t *data, uint32_t len, uint16_t type) { - *(statelensub_t *)data = len; - data += sizeof(statelensub_t); - *(uint32_t *)data = (DHT_STATE_COOKIE_TYPE << 16) | type; - data += sizeof(uint32_t); + uint32_t *data32 = (uint32_t *)data; + data32[0] = len; + data32[1] = (DHT_STATE_COOKIE_TYPE << 16) | type; + data += sizeof(uint32_t) * 2; return data; } /* Save the DHT in data where data is an array of size DHT_size(). */ void DHT_save(DHT *dht, uint8_t *data) { - statelensub_t len; + uint32_t len; uint16_t type; *(uint32_t *)data = DHT_STATE_COOKIE_GLOBAL; data += sizeof(uint32_t); diff --git a/toxcore/Messenger.c b/toxcore/Messenger.c index 682cb1be..2fbb1d3f 100644 --- a/toxcore/Messenger.c +++ b/toxcore/Messenger.c @@ -1361,13 +1361,10 @@ static int Messenger_load_old(Messenger *m, uint8_t *data, uint32_t length) #define MESSENGER_STATE_TYPE_FRIENDS 3 #define MESSENGER_STATE_TYPE_NAME 4 -typedef uint16_t statelensub_t; - /* return size of the messenger data (for saving) */ uint32_t Messenger_size(Messenger *m) { - uint32_t size32 = sizeof(uint32_t), lengthsublen = sizeof(statelensub_t); - uint32_t sizesubhead = lengthsublen + size32; + uint32_t size32 = sizeof(uint32_t), sizesubhead = size32 * 2; return size32 * 2 // global cookie + sizesubhead + sizeof(uint32_t) + crypto_box_PUBLICKEYBYTES + crypto_box_SECRETKEYBYTES + sizesubhead + DHT_size(m->dht) // DHT @@ -1376,19 +1373,19 @@ uint32_t Messenger_size(Messenger *m) ; } -static uint8_t *z_state_save_subheader(uint8_t *data, statelensub_t len, uint16_t type) +static uint8_t *z_state_save_subheader(uint8_t *data, uint32_t len, uint16_t type) { - *(statelensub_t *)data = len; - data += sizeof(statelensub_t); - *(uint32_t *)data = (MESSENGER_STATE_COOKIE_TYPE << 16) | type; - data += sizeof(uint32_t); + uint32_t *data32 = (uint32_t *)data; + data32[0] = len; + data32[1] = (MESSENGER_STATE_COOKIE_TYPE << 16) | type; + data += sizeof(uint32_t) * 2; return data; } /* Save the messenger in data of size Messenger_size(). */ void Messenger_save(Messenger *m, uint8_t *data) { - statelensub_t len; + uint32_t len; uint16_t type; uint32_t *data32, size32 = sizeof(uint32_t); @@ -1484,17 +1481,15 @@ static int messenger_load_state_callback(void *outer, uint8_t *data, uint32_t le int Messenger_load(Messenger *m, uint8_t *data, uint32_t length) { uint32_t cookie_len = 2 * sizeof(uint32_t); - if (length > cookie_len) { - uint32_t *data32 = (uint32_t *)data; - if (!data32[0] && (data32[1] == MESSENGER_STATE_COOKIE_GLOBAL)) { - return load_state(messenger_load_state_callback, m, data + cookie_len, - length - cookie_len, MESSENGER_STATE_COOKIE_TYPE); - } - - /* old state file or too short */ - } + if (length < cookie_len) + return -1; - return Messenger_load_old(m, data, length); + uint32_t *data32 = (uint32_t *)data; + if (!data32[0] && (data32[1] == MESSENGER_STATE_COOKIE_GLOBAL)) + return load_state(messenger_load_state_callback, m, data + cookie_len, + length - cookie_len, MESSENGER_STATE_COOKIE_TYPE); + else /* old state file */ + return Messenger_load_old(m, data, length); } /* Allocate and return a list of valid friend id's. List must be freed by the diff --git a/toxcore/util.c b/toxcore/util.c index 55b51709..653e8d5f 100644 --- a/toxcore/util.c +++ b/toxcore/util.c @@ -56,13 +56,13 @@ int load_state(load_state_callback_func load_state_callback, void *outer, return -1; } - state_length_sub_t length_sub; + uint16_t type; - uint32_t size32 = sizeof(uint32_t), length_sub_len = sizeof(state_length_sub_t); - uint32_t size_head = length_sub_len + size32, cookie_type; + uint32_t length_sub, cookie_type; + uint32_t size32 = sizeof(uint32_t), size_head = size32 * 2; while (length > size_head) { - length_sub = *(state_length_sub_t *)data; - cookie_type = *(uint32_t *)(data + length_sub_len); + length_sub = *(uint32_t *)data; + cookie_type = *(uint32_t *)(data + size32); data += size_head; length -= size_head; diff --git a/toxcore/util.h b/toxcore/util.h index 41f558c1..00482862 100644 --- a/toxcore/util.h +++ b/toxcore/util.h @@ -14,7 +14,6 @@ bool ipp_eq(IP_Port a, IP_Port b); bool id_eq(uint8_t *dest, uint8_t *src); void id_cpy(uint8_t *dest, uint8_t *src); -typedef uint16_t state_length_sub_t; typedef int (*load_state_callback_func)(void *outer, uint8_t *data, uint32_t len, uint16_t type); int load_state(load_state_callback_func load_state_callback, void *outer, uint8_t *data, uint32_t length, uint16_t cookie_inner); -- cgit v1.2.3