dnrops
/
GeoIP2-swift
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

I couldn't let this whole repo go. Eventually I solved the slow index buildup with a lazy normalizataion & solved the one last blocker of properly querying the databases. From a functionality perspective I guess it's complete. Now I gotta: 

- clean up the tests
 - refactor parts of the source
 - find additional optimization opportunities. I don't like the slow init.
 - document
 - tell the world it's done

Signed-off-by: Adam Rocska <adam.rocska@adams.solutions>
This commit is contained in:
Adam Rocska 2020-06-01 12:11:54 +02:00
parent a1703d884e
commit aa6d2958c7
3 changed files with 15 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Sources/DataSection/InMemoryDataSection.swift
View File

@ -53,7 +53,7 @@ public class InMemoryDataSection: DataSection {
} }
public func lookup(pointer: Int) -> [String: Payload]? { public func lookup(pointer: Int) -> [String: Payload]? {
guard let payload = decoder.read(at: pointer, resolvePointers: true) else { return nil } guard let payload = decoder.read(at: pointer - Int(metadata.nodeCount) - 16, resolvePointers: true) else { return nil }
guard case let Payload.map(result) = payload else { return nil } guard case let Payload.map(result) = payload else { return nil }
return result return result
} }

26
Sources/IndexReader/InMemoryIndex.swift
View File

@ -4,9 +4,9 @@ import struct MetadataReader.Metadata
public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Pointer: FixedWidthInteger { public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Pointer: FixedWidthInteger {
private let metadata: Metadata private let metadata: Metadata
private let tree: [Pointer: Node<Pointer>] private let tree: Data
init(metadata: Metadata, tree: [Pointer: Node<Pointer>]) { init(metadata: Metadata, tree: Data) {
self.metadata = metadata self.metadata = metadata
self.tree = tree self.tree = tree
} }
@ -23,20 +23,14 @@ public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Point
let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: searchTreeSize) let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: searchTreeSize)
let read = stream.read(buffer, maxLength: searchTreeSize) let read = stream.read(buffer, maxLength: searchTreeSize)
var tree: [Pointer: Node<Pointer>] = [:] buffer.deallocate()
let nodes = Data( stream.close()
// precondition(tree.count == metadata.nodeCount)
self.init(metadata: metadata, tree: Data(
bytesNoCopy: buffer, bytesNoCopy: buffer,
count: read, count: read,
deallocator: .none deallocator: .none
) ))
.chunked(into: Int(metadata.nodeByteSize))
.map({ Node<Pointer>($0) })
for (pointer, node) in nodes.enumerated() { tree[Pointer(pointer)] = node }
buffer.deallocate()
stream.close()
precondition(tree.count == metadata.nodeCount)
self.init(metadata: metadata, tree: tree)
} }
public func lookup(_ ip: IpAddress) -> Pointer? { public func lookup(_ ip: IpAddress) -> Pointer? {
@ -50,9 +44,13 @@ public class InMemoryIndex<Pointer>: Index where Pointer: UnsignedInteger, Point
} }
var pointer: Pointer = 0 var pointer: Pointer = 0
let nodeByteSize = Pointer.init(metadata.nodeByteSize)
while pointer < metadata.nodeCount { while pointer < metadata.nodeCount {
guard let direction = stack.popLast() else { break } guard let direction = stack.popLast() else { break }
guard let node = tree[pointer] else { return pointer } let pointerInTree = pointer * nodeByteSize
let nodeCandidate = tree[pointerInTree..<pointerInTree + nodeByteSize]
let node = Node<Pointer>(nodeCandidate)
// guard let node = tree[pointer] else { return pointer }
switch direction { switch direction {
case .left: pointer = node.left case .left: pointer = node.left

3
Tests/DBReaderTests/InMemoryReaderTest.swift
View File

@ -83,7 +83,8 @@ class InMemoryReaderTest: XCTestCase {
let factory = ReaderFactory() let factory = ReaderFactory()
let streamFactory: () -> InputStream = { let streamFactory: () -> InputStream = {
InputStream( InputStream(
fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/ApiTests/Resources/GeoLite2-City_20200526/GeoLite2-City.mmdb" // fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/ApiTests/Resources/GeoLite2-City_20200526/GeoLite2-City.mmdb"
fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/ApiTests/Resources/GeoLite2-ASN_20200526/GeoLite2-ASN.mmdb"
// fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/DBReaderTests/Resources/GeoLite2-Country_20200421/GeoLite2-Country.mmdb" // fileAtPath: "/Users/rocskaadam/src/adam-rocska/src/GeoIP2-swift/Tests/DBReaderTests/Resources/GeoLite2-Country_20200421/GeoLite2-Country.mmdb"
)! )!
} }