1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
extern crate blist;
extern crate ilc_base;
mod ageset;
pub mod freq;
/// No-op log parsing
pub mod parse {
use ilc_base::{self, Context, Decode};
use std::io::BufRead;
/// Simply parse the input, without further validation or conversion. No information is stored.
/// This will return `Err` if the decoder yields `Err`.
pub fn parse(ctx: &Context, input: &mut BufRead, decoder: &mut Decode) -> ilc_base::Result<()> {
for e in decoder.decode(&ctx, input) {
try!(e);
}
Ok(())
}
}
/// Log format conversion
pub mod convert {
use ilc_base::{self, Context, Decode, Encode};
use std::io::{BufRead, Write};
/// Convert from one format to another, not necessarily different, format. In combination with a
/// timezone offset, this can be used to correct the timestamps.
/// Will return `Err` and abort conversion if the decoder yields `Err` or re-encoding fails.
pub fn convert(ctx: &Context,
input: &mut BufRead,
decoder: &mut Decode,
output: &mut Write,
encoder: &Encode)
-> ilc_base::Result<()> {
for e in decoder.decode(&ctx, input) {
try!(encoder.encode(&ctx, output, &try!(e)));
}
Ok(())
}
}
/// Last-seen of nicks
pub mod seen {
use ilc_base::{self, Context, Decode, Encode, Event};
use std::io::{BufRead, Write};
/// Return the last message of a given nickname, searching from the beginning of the logs.
/// Will return `Err` if the decoder yields `Err`. This relies on absolute timestamps, and
/// behaviour without full dates is undefined.
pub fn seen(nick: &str,
ctx: &Context,
input: &mut BufRead,
decoder: &mut Decode,
output: &mut Write,
encoder: &Encode)
-> ilc_base::Result<()> {
let mut last: Option<Event> = None;
for e in decoder.decode(&ctx, input) {
let m: Event = try!(e);
if m.ty.involves(nick) &&
last.as_ref().map_or(true,
|last| m.time.as_timestamp() > last.time.as_timestamp()) {
last = Some(m)
}
}
if let Some(ref m) = last {
try!(encoder.encode(&ctx, output, m));
}
Ok(())
}
}
/// Internal (as opposed to external, not to be confused with private) log sorting
pub mod sort {
use ilc_base::{self, Context, Decode, Encode, Event};
use std::io::{BufRead, Write};
/// **Memory-intensive**
/// Sort the input, discarding faulty events. This will
/// read *all events* into memory, then sort them by time and write them back.
/// Behaviour is undefined if events lack full date information.
///
/// *This should be an external merge-sort, but is a placeholder until implementation*
pub fn sort(ctx: &Context,
input: &mut BufRead,
decoder: &mut Decode,
output: &mut Write,
encoder: &Encode)
-> ilc_base::Result<()> {
let mut events: Vec<Event> = decoder.decode(&ctx, input)
.flat_map(Result::ok)
.collect();
events.sort_by(|a, b| a.time.cmp(&b.time));
for e in events {
try!(encoder.encode(&ctx, output, &e));
}
Ok(())
}
}
/// Event deduplication
pub mod dedup {
use std::io::{BufRead, Write};
use std::hash::{Hash, Hasher};
use ageset::AgeSet;
use ilc_base::{self, Context, Decode, Encode, Event};
#[derive(Clone, Debug, PartialEq, Eq)]
struct NoTimeHash<'a>(pub Event<'a>);
impl<'a> Hash for NoTimeHash<'a> {
fn hash<H>(&self, state: &mut H)
where H: Hasher
{
self.0.ty.hash(state);
self.0.channel.hash(state);
}
}
/// Deduplicate subsequent identical elements, e.g. after a sorting
/// operation. This will **not** read all events into memory, and only
/// operate on a short window of events. Therefore, it'll only work correctly
/// on sorted or very short logs.
pub fn dedup(ctx: &Context,
input: &mut BufRead,
decoder: &mut Decode,
output: &mut Write,
encoder: &Encode)
-> ilc_base::Result<()> {
let mut backlog = AgeSet::new();
for e in decoder.decode(&ctx, input) {
if let Ok(e) = e {
let newest_event = e.clone();
backlog.prune(move |a: &NoTimeHash| {
let age = newest_event.time.as_timestamp() - a.0.time.as_timestamp();
age > 5000
});
// write `e` if it's a new event
let n = NoTimeHash(e);
if !backlog.contains(&n) {
try!(encoder.encode(&ctx, output, &n.0));
backlog.push(n);
}
}
}
Ok(())
}
}
|