I couldn't let this whole repo go. Eventually I solved the slow index buildup with a lazy normalizataion & solved the one last blocker of properly querying the databases. From a functionality perspective I guess it's complete. Now I gotta:
- clean up the tests - refactor parts of the source - find additional optimization opportunities. I don't like the slow init. - document - tell the world it's done Signed-off-by: Adam Rocska <adam.rocska@adams.solutions>
This commit is contained in:
parent
a1703d884e
commit
aa6d2958c7
|
@ -53,7 +53,7 @@ public class InMemoryDataSection: DataSection {
|
||||||
}
|
}
|
||||||
|
|
||||||
public func lookup(pointer: Int) -> [String: Payload]? {
|
public func lookup(pointer: Int) -> [String: Payload]? {
|
||||||
guard let payload = decoder.read(at: pointer, resolvePointers: true) else { return nil }
|
guard let payload = decoder.read(at: pointer - Int(metadata.nodeCount) - 16, resolvePointers: true) else { return nil }
|
||||||
guard case let Payload.map(result) = payload else { return nil }
|
guard case let Payload.map(result) = payload else { return nil }
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,9 +4,9 @@ import struct MetadataReader.Metadata
|
||||||
public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Pointer: FixedWidthInteger {
|
public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Pointer: FixedWidthInteger {
|
||||||
|
|
||||||
private let metadata: Metadata
|
private let metadata: Metadata
|
||||||
private let tree: [Pointer: Node<Pointer>]
|
private let tree: Data
|
||||||
|
|
||||||
init(metadata: Metadata, tree: [Pointer: Node<Pointer>]) {
|
init(metadata: Metadata, tree: Data) {
|
||||||
self.metadata = metadata
|
self.metadata = metadata
|
||||||
self.tree = tree
|
self.tree = tree
|
||||||
}
|
}
|
||||||
|
@ -23,20 +23,14 @@ public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Point
|
||||||
let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: searchTreeSize)
|
let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: searchTreeSize)
|
||||||
let read = stream.read(buffer, maxLength: searchTreeSize)
|
let read = stream.read(buffer, maxLength: searchTreeSize)
|
||||||
|
|
||||||
var tree: [Pointer: Node<Pointer>] = [:]
|
buffer.deallocate()
|
||||||
let nodes = Data(
|
stream.close()
|
||||||
|
// precondition(tree.count == metadata.nodeCount)
|
||||||
|
self.init(metadata: metadata, tree: Data(
|
||||||
bytesNoCopy: buffer,
|
bytesNoCopy: buffer,
|
||||||
count: read,
|
count: read,
|
||||||
deallocator: .none
|
deallocator: .none
|
||||||
)
|
))
|
||||||
.chunked(into: Int(metadata.nodeByteSize))
|
|
||||||
.map({ Node<Pointer>($0) })
|
|
||||||
for (pointer, node) in nodes.enumerated() { tree[Pointer(pointer)] = node }
|
|
||||||
|
|
||||||
buffer.deallocate()
|
|
||||||
stream.close()
|
|
||||||
precondition(tree.count == metadata.nodeCount)
|
|
||||||
self.init(metadata: metadata, tree: tree)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public func lookup(_ ip: IpAddress) -> Pointer? {
|
public func lookup(_ ip: IpAddress) -> Pointer? {
|
||||||
|
@ -50,9 +44,13 @@ public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Point
|
||||||
}
|
}
|
||||||
var pointer: Pointer = 0
|
var pointer: Pointer = 0
|
||||||
|
|
||||||
|
let nodeByteSize = Pointer.init(metadata.nodeByteSize)
|
||||||
while pointer < metadata.nodeCount {
|
while pointer < metadata.nodeCount {
|
||||||
guard let direction = stack.popLast() else { break }
|
guard let direction = stack.popLast() else { break }
|
||||||
guard let node = tree[pointer] else { return pointer }
|
let pointerInTree = pointer * nodeByteSize
|
||||||
|
let nodeCandidate = tree[pointerInTree..<pointerInTree + nodeByteSize]
|
||||||
|
let node = Node<Pointer>(nodeCandidate)
|
||||||
|
// guard let node = tree[pointer] else { return pointer }
|
||||||
|
|
||||||
switch direction {
|
switch direction {
|
||||||
case .left: pointer = node.left
|
case .left: pointer = node.left
|
||||||
|
|
|
@ -83,7 +83,8 @@ class InMemoryReaderTest: XCTestCase {
|
||||||
let factory = ReaderFactory()
|
let factory = ReaderFactory()
|
||||||
let streamFactory: () -> InputStream = {
|
let streamFactory: () -> InputStream = {
|
||||||
InputStream(
|
InputStream(
|
||||||
fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/ApiTests/Resources/GeoLite2-City_20200526/GeoLite2-City.mmdb"
|
// fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/ApiTests/Resources/GeoLite2-City_20200526/GeoLite2-City.mmdb"
|
||||||
|
fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/ApiTests/Resources/GeoLite2-ASN_20200526/GeoLite2-ASN.mmdb"
|
||||||
// fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/DBReaderTests/Resources/GeoLite2-Country_20200421/GeoLite2-Country.mmdb"
|
// fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/DBReaderTests/Resources/GeoLite2-Country_20200421/GeoLite2-Country.mmdb"
|
||||||
)!
|
)!
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue