x86_64_sse2_x87/pbf/blk/primblk/primgrp/relations/text.s

   1 ; vim: set filetype=fasm foldmethod=marker commentstring=;%s colorcolumn=101 :
   2 ; XXX: We actually used this code unit as a training ground. Expect it to be very overkill.
   3 relations: namespace relations
   4 ;{{{ init_once -------------------------------------------------------------------------------------
   5 align_nops
   6 init_once:
   7         TRACE 'PBF:BLK:PRIMBLK:PRIMGRP:RELATIONS:INIT_ONCE'
   8         mov rbp, rsp
   9         and rsp, not 0xf
  10
  11         xor edi, edi
  12         mov rsi, array.bytes_n
  13         mov rdx, 1b or 10b      ; PROT_READ  | PROT_WRITE
  14         mov r10, 0x22           ; MAP_PRIVATE | MAP_ANONYMOUS
  15         xor r8d, r8d
  16         xor r9d, r9d
  17         mov rax, linux.mmap
  18         syscall
  19         cmp rax, linux.errno.last
  20                 jae .err_relations_mmap_failed
  21         mov [array], rax
  22         mov [array.finish], rax
  23
  24         lea rdi, [msg.idx_relations_n]
  25         mov rsi, array.idx.n
  26         call qword [libc.printf]
  27
  28         xor edi, edi
  29         mov rsi, array.idx.bytes_n
  30         mov rdx, 1b or 10b      ; PROT_READ  | PROT_WRITE
  31         mov r10, 0x22           ; MAP_PRIVATE | MAP_ANONYMOUS
  32         xor r8d, r8d
  33         xor r9d, r9d
  34         mov rax, linux.mmap
  35         syscall
  36         cmp rax, linux.errno.last
  37                 jae .err_relations_idx_mmap_failed
  38         mov [array.idx], rax
  39         add rax, 8 * 3
  40         mov [array.idx.finish], rax ; next availabe idx slot
  41
  42         stc ; return true
  43         mov rsp, rbp
  44         ret
  45 .err_relations_mmap_failed:
  46         mov rdi, 2 ; stderr
  47         mov rsi, rax ; errno
  48         lea rsi, [msg.mmap_failed]
  49         call qword [libc.dprintf]
  50         clc ; return false
  51         mov rsp, rbp
  52         ret
  53 .err_relations_idx_mmap_failed:
  54         mov rdi, 2 ; stderr
  55         mov rsi, rax ; errno
  56         lea rsi, [msg.idx_mmap_failed]
  57         call qword [libc.dprintf]
  58         clc ; return false
  59         mov rsp, rbp
  60         ret
  61 ;}}} init_once -------------------------------------------------------------------------------------
  62 ; local memcpy ABI
  63 define memcpy_dst               rbx     ; memcpy_dst(out) = memcpy_dst(in) + memcpy_bytes_n
  64 define memcpy_src               r11     ; clobbered, don't care
  65 define memcpy_bytes_n           r9      ; clobbered, don't care
  66 define memcpy_link              rcx     ; the return addr
  67 ;===================================================================================================
  68 ;{{{ parse -----------------------------------------------------------------------------------------
  69 ; scratch ~ rax rcx r11 r9
  70 ; from primgrp
  71 define relations_p              rbx     ; in
  72 define relations_finish_p       r13     ; in
  73 define flags                    r15     ; in/out
  74 define strtbl_idx_p             r8
  75 define lat_of_lon_of            xmm3    ; f64/f64
  76 ; local
  77 define id                       r12
  78 define keys_p                   r14     ; can be 0
  79 define keys_p_d                 r14d    ; can be 0
  80 define keys_finish_p            rbp
  81 define vals_p                   r10     ; num of vals is num of keys
  82 define memids_p                 rdx     ; memids = MEMber IDs
  83 define memids_p_d               edx
  84 define memids_finish_p          rsi
  85 define types_p                  rdi     ; num of types is num of memids
  86 define roles_p_xmm              xmm2    ; the role of a member is a string
  87 align_nops
  88 parse:
  89         xor keys_p_d, keys_p_d ; if no (key/val)
  90         xor memids_p_d, memids_p_d ; presume we can have no memids (should not happen though)
  91 align_nops
  92 .next_msg_key:
  93         ; "relationS" is plural because of the primgrp field name, but there is
  94         ; actually only one "relation" which will be repeated
  95         cmp relations_p, relations_finish_p
  96                 je unserialize
  97         varint_load r11, relations_p ; msg key
  98         mov r9, r11 ; = msg key copy
  99         shr r11, 3 ; = field num
 100         cmp r11, 1
 101                 je .id_found
 102         cmp r11, 2
 103                 je .keys_found
 104         cmp r11, 3
 105                 je .vals_found
 106         cmp r11, 8
 107                 je .roles_found
 108         cmp r11, 9
 109                 je .memids_found
 110         cmp r11, 10
 111                 je .types_found
 112         and r9, 111b ; field type
 113         TRACE 'PBF:BLK:PRIMBLK:PRIMGRP:RELATIONS:MSG:FIELD %lu OF TYPE %lu', r11, r9
 114         val_skip relations_p, r9, r11
 115         jmp .next_msg_key
 116 align_nops
 117 .id_found:
 118         varint_load id, relations_p
 119         jmp .next_msg_key
 120 align_nops
 121 .keys_found:
 122         varint_load r11, relations_p ; keys_sz
 123         mov keys_p, relations_p
 124         TRACE 'PBF:BLK:PRIMBLK:PRIMGRP:RELATIONS:KEYS:start = %p:size = %lu(0x%lx) bytes', relations_p, r11, r11
 125         add relations_p, r11
 126         mov keys_finish_p, relations_p
 127         jmp .next_msg_key
 128 align_nops
 129 .vals_found:
 130         varint_load r11, relations_p ; vals_sz
 131         mov vals_p, relations_p
 132         TRACE 'PBF:BLK:PRIMBLK:PRIMGRP:RELATIONS:VALS:start = %p:size = %lu(0x%lx) bytes', vals_p, r11, r11
 133         add relations_p, r11
 134         jmp .next_msg_key
 135 align_nops
 136 .memids_found:
 137         varint_load r11, relations_p ; memids_sz
 138         mov memids_p, relations_p
 139         TRACE 'PBF:BLK:PRIMBLK:PRIMGRP:RELATIONS:MEMIDS:start = %p:size = %lu(0x%lx) bytes', memids_p, r11, r11
 140         add relations_p, r11
 141         mov memids_finish_p, relations_p
 142         jmp .next_msg_key
 143 align_nops
 144 .types_found:
 145         varint_load r11, relations_p ; types_sz
 146         mov types_p, relations_p
 147         TRACE 'PBF:BLK:PRIMBLK:PRIMGRP:RELATIONS:TYPES:start = %p:size = %lu(0x%lx) bytes', types_p, r11, r11
 148         add relations_p, r11
 149         jmp .next_msg_key
 150 align_nops
 151 .roles_found:
 152         varint_load r11, relations_p ; roles_sz
 153         TRACE 'PBF:BLK:PRIMBLK:PRIMGRP:RELATIONS:ROLES:start = %p:size = %lu(0x%lx) bytes', relations_p, r11, r11
 154         movq roles_p_xmm, relations_p
 155         add relations_p, r11
 156         jmp .next_msg_key
 157 purge relations_finish_p
 158 purge relations_p
 159 ;}}} parse -----------------------------------------------------------------------------------------
 160 ;{{{ unserialize -----------------------------------------------------------------------------------
 161 ; scratch ~ rax rcx r11 r9
 162 ; from primgrp
 163 ;define flags                   r15     ; in/out
 164 ;define strtbl_idx_p            r8
 165 ;define lat_of_lon_of           xmm3    ; f64/f64
 166 ; local from parse
 167 ;define id                      r12
 168 ;define keys_p                  r14     ; can be 0
 169 ;define keys_p_d                        r14d    ; can be 0
 170 ;define keys_finish_p           rbp
 171 ;define vals_p                  r10     ; num of vals is num of keys
 172 ;define memids_p                rdx     ; memids = MEMber IDs
 173 ;define memids_p_d              edx
 174 ;define memids_finish_p         rsi
 175 ;define types_p                 rdi     ; num of types is num of memids
 176 ;define roles_p_xmm             xmm2    ; the role of a member is a string
 177 ; local
 178 define relation_p               rbx
 179 define relation_start_p         r13
 180 align_nops
 181 unserialize:
 182         mov relations_p, [array.finish]
 183         mov relation_start_p, relations_p
 184 ;{{{ keys_val_cpy ----------------------------------------------------------------------------------
 185 keys_vals_cpy:
 186         add relation_p, array.relation_t.keys_vals ; relation_p points on the start of the keys_vals section
 187         test keys_p, keys_p
 188                 jz unserialize.keys_vals_done
 189         jmp .entry
 190 align_nops
 191 .next_key_val:
 192         cmp keys_p, keys_finish_p
 193                 je unserialize.keys_vals_done
 194 .entry:
 195         varint_load r11, keys_p ; = key_id, keys_p points on next key_id
 196         mov r11, [strtbl_idx_p + 8 * r11] ; = key_src_addr
 197         varint_load r9, r11 ; r9 = key_bytes_n, r11 = key_src_addr
 198         mov [relation_p], r9b ; store the byte containing the sz of the key str in bytes
 199         inc relation_p
 200         ; memcpy_dst = relation_p(rbx), memcpy_src = key_src_addr(r11), memcpy_bytes_n = key_bytes_n(r9)
 201         lea memcpy_link, [.key_cpy_done]
 202         jmp memcpy ; relation_p = relation_p + key_bytes_n
 203 align_nops
 204 .key_cpy_done:
 205         varint_load r11, vals_p ; = val_id, vals_p points on next val_id
 206         mov r11, [strtbl_idx_p + 8 * r11] ; = val_src_addr
 207         varint_load r9, r11; r9 = val_bytes_n, r11 = val_src_addr
 208         mov [relation_p], r9w ; store the short containing the sz of the val str in bytes
 209         add relation_p, 2
 210         ; memcpy_dst = relation_p(rbx), memcpy_src = val_src_addr(r11), memcpy_bytes_n = val_bytes_n(r9)
 211         lea memcpy_link, [.next_key_val]
 212         jmp memcpy ; relation_p = relation_p + key_bytes_n
 213 ;}}} keys_val_cpy ----------------------------------------------------------------------------------
 214 align_nops
 215 unserialize.keys_vals_done:
 216         mov byte [relation_p], 0 ; insert the 0-szed key/terminator
 217         inc relation_p
 218         ; here relation_p points on the start of the section of members
 219         mov [relation_start_p + array.relation_t.members], relation_p
 220         test memids_p, memids_p
 221                 jz unserialize.members_done ; unlikely
 222 purge vals_p
 223 purge keys_finish_p
 224 purge keys_p_d
 225 purge keys_p
 226 ;{{{ members ---------------------------------------------------------------------------------------
 227 ; scratch ~ rax rcx r11 r9
 228 ; from primgrp
 229 ;define flags                   r15     ; in/out
 230 ;define strtbl_idx_p            r8
 231 ;define lat_of_lon_of           xmm3    ; f64/f64
 232 ; local from parse
 233 ;define id                      r12
 234 ;define memids_p                rdx     ; memids = MEMber IDs
 235 ;define memids_p_d              edx
 236 ;define memids_finish_p         rsi
 237 ;define types_p                 rdi     ; num of types is num of memids
 238 ;define roles_p_xmm             xmm2    ; the role of a member is a string
 239 ; local from unserialize
 240 ;define relation_p              rbx
 241 ;define relation_start_p        r13
 242 ; local
 243 define roles_p                  r14     ; get it from xmm roles_p_xmm
 244 define prev_memid               rbp
 245 define prev_memid_d             ebp
 246 define member_p                 relation_p ; alias
 247 define member_p.p               member_p + array.relation_t.member_t.p
 248 define member_p.type            member_p + array.relation_t.member_t.type
 249 members:
 250         movq roles_p, roles_p_xmm
 251         xor prev_memid_d, prev_memid_d ; prev_id = 0
 252         jmp .entry
 253 align_nops
 254 .next_member:
 255         cmp memids_p, memids_finish_p
 256                 je unserialize.members_done
 257 .entry:
 258         ; decode the relation id -------------------------------------------------------------------
 259         varint_load r11, memids_p ; load/skip the raw zigzag delta_refs
 260         mov rax, r11
 261         shr r11, 1 ; i >> 1
 262         and rax, 1 ; i & 1
 263         neg rax ; -(i & 1)
 264         xor r11, rax ; (i >> 1) ^ -(i & 1)  ; = delta_id
 265         add prev_memid, r11 ; = prev_memid = memid because DELTA encoded
 266         mov [member_p.p], prev_memid ; store memid
 267         ; decode the type --------------------------------------------------------------------------
 268         varint_load r11, types_p
 269         mov byte [member_p.type], r11b ; a byte in enough
 270         ; cpy the role str -------------------------------------------------------------------------
 271         add member_p, array.relation_t.member_t.role ; member_p points on the first byte of the role str
 272         varint_load r11, roles_p ; = role_str_id
 273         mov r11, [strtbl_idx_p + 8 * r11] ; role_src_addr
 274         varint_load r9, r11 ; r9 = role_bytes_n, r11 = role_src_addr
 275         mov [member_p], r9b ; insert the byte containing the sz of the role str in bytes
 276         inc member_p
 277         ; memcpy_dst = member_p(rbx), memcpy_src = role_src_addr(r11), memcpy_bytes_n = role_bytes_n(r9)
 278         lea memcpy_link, [.next_member]
 279         jmp memcpy ; member_p = member_p + role_bytes_n
 280 purge member_p.type
 281 purge member_p.p
 282 purge member_p
 283 purge prev_memid_d
 284 purge prev_memid
 285 purge roles_p
 286 ;}}} members ---------------------------------------------------------------------------------------
 287 purge roles_p_xmm
 288 purge memids_finish_p
 289 purge memids_p_d
 290 purge memids_p
 291 purge types_p
 292 unserialize.members_done:
 293         ; here relation_p points on the terminating member (only a 0xff type and nothing else)
 294         mov byte [relation_p], 0xff ; the terminator type is at the start of relation_p
 295         inc relation_p ; = next relation
 296         mov [relation_start_p + array.relation_t.next], relation_p
 297         mov [array.finish], relation_p
 298 purge relation_p
 299 ;{{{ idx_insert_node -------------------------------------------------------------------------------
 300 ; scratch ~ rax
 301 ; from primgrp
 302 ;define flags                   r15     ; in/out
 303 ;define strtbl_idx_p            r8
 304 ;define lat_of_lon_of           xmm3    ; f64/f64
 305 ; local from parse
 306 ;define id                      r12
 307 ; local from unserialize
 308 ;define relation_start_p        r13
 309 ; local
 310 define idx_finish_p             rbx
 311 define id_bit_idx               r14
 312 define id_bit_idx_d             r14d
 313 define bit_val_select           rcx ; we use rcx here because we may use rcx compressed instructions
 314 define bit_val_select_d         ecx
 315 define bit_val_select_b         cl
 316 define id_bit_idx_msb           rbp
 317 define idx_slot_p               r9
 318 ; TODO:REMOVE
 319 ;define idx_slot_p.relation     idx_slot_p + array.idx.slot_t.relation
 320 idx_insert_node:
 321         mov idx_slot_p, [array.idx]
 322         mov idx_finish_p, [array.idx.finish]
 323         TRACE_RELATIONS_IDX 'id=%#016lx nodes.idx=%p nodes.finish=%p', id, idx_slot_p, idx_finish_p
 324         xor id_bit_idx_d, id_bit_idx_d
 325         xor bit_val_select_d, bit_val_select_d
 326         mov id_bit_idx_msb, -1 ; this is to make it work with id = 0
 327         bsr id_bit_idx_msb, id ; if id = 0, id_bit_idx_msb is untouched namely -1
 328         TRACE_RELATIONS_IDX 'id_bit_idx_msb=%lu', id_bit_idx_msb
 329 align_nops
 330 .next_id_bit:
 331         TRACE_RELATIONS_IDX 'id_bit_idx=%lu', id_bit_idx
 332         cmp id_bit_idx_msb, id_bit_idx
 333                 jb .insert_here
 334         bt id, id_bit_idx
 335         setc bit_val_select_b ; bit_val_select = 1 if id[id_bit_idx] = 1, else 0
 336         TRACE_RELATIONS_IDX 'bit_val_select=%lu', bit_val_select
 337         mov rax, [idx_slot_p + 8 * bit_val_select]
 338         TRACE_RELATIONS_IDX 'slot=%p bit slot=%p', idx_slot_p, rax
 339         test rax, rax
 340                 jnz .idx_existing_slot
 341         TRACE_RELATIONS_IDX 'non existing slot using finish=%p', idx_finish_p
 342         mov rax, idx_finish_p
 343         add idx_finish_p, array.idx.slot_t.bytes_n ; we could zero the mem here, but mmap does it for us
 344 if ~SYSTEM_LEGACY
 345         prefetchw [idx_finish_p + 64 * 2]       ; try to get ready
 346         prefetchw [idx_finish_p + 64 * 3]
 347 end if
 348         mov [idx_slot_p + 8 * bit_val_select], rax
 349 align_nops
 350 .idx_existing_slot:
 351         mov idx_slot_p, rax
 352         TRACE_RELATIONS_IDX 'next_slot=%p', idx_slot_p
 353         inc id_bit_idx
 354
 355         jmp .next_id_bit
 356 align_nops
 357 .insert_here:
 358 if TRACE_RELATIONS_IDX_ENABLED
 359         mov rax, [idx_slot_p + array.idx.slot_t.relation]
 360         test rax, rax
 361                 jz .relation_slot_available
 362         TRACE_RELATIONS_IDX 'WARNING: overwritting an existing relation=%p', rax
 363 .relation_slot_available:
 364         TRACE_RELATIONS_IDX 'inserting relation=%p in slot=%p', relation_start_p, idx_slot_p
 365 end if
 366         mov [idx_slot_p + array.idx.slot_t.relation], relation_start_p
 367         ; XXX: if one day we need to be backed by a file we will need to DONT_NEED "madvise" the
 368         mov [array.idx.finish], idx_finish_p
 369         ; the current relation memory
 370 purge idx_slot_p
 371 purge id_bit_idx_msb
 372 purge bit_val_select_b
 373 purge bit_val_select_d
 374 purge bit_val_select
 375 purge id_bit_idx_d
 376 purge id_bit_idx
 377 purge idx_finish_p
 378 ;}}} idx_insert_node -------------------------------------------------------------------------------
 379 purge relation_start_p
 380 purge id
 381 purge lat_of_lon_of
 382 purge strtbl_idx_p
 383 purge flags
 384         jmp primgrp.parse.return_from_relations
 385 ;}}} unserialize -----------------------------------------------------------------------------------
 386 ;{{{ local memcpy(sse) -----------------------------------------------------------------------------
 387 ; XXX: lddqu performs aligned reads, carefull where you use this, but should be fine with classic
 388 ; paginated memory.
 389 ; scratch ~ rax xmm7 xmm6 xmm5 xmm4
 390 align_nops
 391 memcpy:
 392         cmp memcpy_bytes_n, 16 * 4
 393                 jb .below64
 394         ; since we can be cache line mis-aligned, speculatively do prefetch 2 cls ahead
 395         prefetchnta [memcpy_src + 64 * 2]  ; speculative non-temporal "3rd" cache line ahead
 396         prefetchnta [memcpy_src + 64 * 3]  ; speculative non-temporal "4th" cache line ahead
 397 if ~SYSTEM_LEGACY
 398         prefetchw [memcpy_dst + 64 * 2]
 399         prefetchw [memcpy_dst + 64 * 3]
 400 end if
 401         lddqu xmm7, [memcpy_src + 16 * 0]
 402         lddqu xmm6, [memcpy_src + 16 * 1]
 403         lddqu xmm5, [memcpy_src + 16 * 2]
 404         lddqu xmm4, [memcpy_src + 16 * 3]
 405
 406         movdqu [memcpy_dst + 16 * 0], xmm7
 407         movdqu [memcpy_dst + 16 * 1], xmm6
 408         movdqu [memcpy_dst + 16 * 2], xmm5
 409         movdqu [memcpy_dst + 16 * 3], xmm4
 410
 411         add memcpy_dst, 16 * 4
 412         sub memcpy_bytes_n, 16 * 4
 413                 jz .done
 414         add memcpy_src, 16 * 4
 415         jmp memcpy
 416 align_nops
 417 .below64:
 418         cmp memcpy_bytes_n, 16 * 3
 419                 jb .below48
 420
 421         lddqu xmm7, [memcpy_src + 16 * 0]
 422         lddqu xmm6, [memcpy_src + 16 * 1]
 423         lddqu xmm5, [memcpy_src + 16 * 2]
 424
 425         movdqu [memcpy_dst + 16 * 0], xmm7
 426         movdqu [memcpy_dst + 16 * 1], xmm6
 427         movdqu [memcpy_dst + 16 * 2], xmm5
 428
 429         add memcpy_dst, 16 * 3
 430         sub memcpy_bytes_n, 16 * 3
 431                 jz .done
 432         add memcpy_src, 16 * 3
 433         ; fall-thru
 434 align_nops
 435 .below48:
 436         cmp memcpy_bytes_n, 16 * 2
 437                 jb .below32
 438
 439         lddqu xmm7, [memcpy_src + 16 * 0]
 440         lddqu xmm6, [memcpy_src + 16 * 1]
 441
 442         movdqu [memcpy_dst + 16 * 0], xmm7
 443         movdqu [memcpy_dst + 16 * 1], xmm6
 444
 445         add memcpy_dst, 16 * 2
 446         sub memcpy_bytes_n, 16 * 2
 447                 jz .done
 448         add memcpy_src, 16 * 2
 449         ; fall-thru
 450 align_nops
 451 .below32:
 452         cmp memcpy_bytes_n, 16 * 1
 453                 jb .below16
 454
 455         lddqu xmm7, [memcpy_src + 16 * 0]
 456         movdqu [memcpy_dst + 16 * 0], xmm7
 457
 458         add memcpy_dst, 16 * 1
 459         sub memcpy_bytes_n, 16 * 1
 460                 jz .done
 461         add memcpy_src, 16 * 1
 462         ; fall-thru
 463 align_nops
 464 .below16:
 465         cmp memcpy_bytes_n, 8 * 1
 466                 jb .below8
 467         mov rax, [memcpy_src + 8 * 0]
 468         mov [memcpy_dst + 8 * 0], rax
 469
 470         add memcpy_dst, 8 * 1
 471         sub memcpy_bytes_n, 8 * 1
 472                 jz .done
 473         add memcpy_src, 8 * 1
 474         ; fall-thru
 475 align_nops
 476 .below8:
 477         cmp memcpy_bytes_n, 7
 478                 je .cpy7
 479         cmp memcpy_bytes_n, 6
 480                 je .cpy6
 481         cmp memcpy_bytes_n, 5
 482                 je .cpy5
 483         cmp memcpy_bytes_n, 4
 484                 je .cpy4
 485         cmp memcpy_bytes_n, 3
 486                 je .cpy3
 487         cmp memcpy_bytes_n, 2
 488                 je .cpy2
 489         cmp memcpy_bytes_n, 1
 490                 je .cpy1
 491         jmp memcpy_link
 492 align_nops
 493 .cpy7:
 494         mov al, [memcpy_src]
 495         mov [memcpy_dst], al
 496         inc memcpy_src
 497         inc memcpy_dst
 498 align_nops
 499 .cpy6:
 500         mov al, [memcpy_src]
 501         mov [memcpy_dst], al
 502         inc memcpy_src
 503         inc memcpy_dst
 504 align_nops
 505 .cpy5:
 506         mov al, [memcpy_src]
 507         mov [memcpy_dst], al
 508         inc memcpy_src
 509         inc memcpy_dst
 510 align_nops
 511 .cpy4:
 512         mov al, [memcpy_src]
 513         mov [memcpy_dst], al
 514         inc memcpy_src
 515         inc memcpy_dst
 516 align_nops
 517 .cpy3:
 518         mov al, [memcpy_src]
 519         mov [memcpy_dst], al
 520         inc memcpy_src
 521         inc memcpy_dst
 522 align_nops
 523 .cpy2:
 524         mov al, [memcpy_src]
 525         mov [memcpy_dst], al
 526         inc memcpy_src
 527         inc memcpy_dst
 528 align_nops
 529 .cpy1:
 530         mov al, [memcpy_src]
 531         mov [memcpy_dst], al
 532         inc memcpy_src
 533         inc memcpy_dst
 534 align_nops
 535 .done:
 536         jmp memcpy_link
 537 ;}}} local memcpy(sse) -----------------------------------------------------------------------------
 538 ; {{{ resolve_ids ----------------------------------------------------------------------------------
 539 ; from pbf.tile_db_build
 540 define flags                    r15
 541 ; local
 542 define relation_p               rbx
 543 define relations_finish_p       r12
 544 define member_p                 r13
 545 define nodes_idx                r14
 546 define ways_idx                 rbp
 547 define relations_idx            r11
 548 define id                       r10
 549 define bit_val_select           r9
 550 define bit_val_select_b         r9b
 551 define id_bit_idx_msb           rcx ; we use rcx for compressed opcodes
 552 define id_bit_idx_msb_d         ecx
 553 define idx_slot_p               r8
 554 align_nops
 555 resolve_ids:
 556         mov relation_p, [array]
 557         mov relations_finish_p, [array.finish]
 558         mov nodes_idx, [primgrp.dense.nodes.idx]
 559         mov ways_idx, [primgrp.ways.array.idx]
 560         mov relations_idx, [primgrp.relations.array.idx]
 561 align_nops
 562 .next_relation:
 563         TRACE_2ND_PASS 'relation=%p relation_finish_p=%p', relation_p, relations_finish_p
 564         cmp relation_p, relations_finish_p
 565                 je pbf.tile_db_build.features_select_pass ; prep 3rd pass
 566         mov member_p, [relation_p + array.relation_t.members]
 567 align_nops
 568 .next_member:
 569         mov al, [member_p + array.relation_t.member_t.type]
 570         ; this field actually contains the id after the first major pass
 571         mov id, [member_p + array.relation_t.member_t.p]
 572         TRACE_2ND_PASS 'member=%p type=%#2lx(lowest byte only) id=%#lx(invalid if type=0xff)', member_p, rax, id
 573         cmp al, 0xff
 574                 jne .have_more_members
 575         mov relation_p, [relation_p + array.relation_t.next]
 576         TRACE_2ND_PASS 'no more members next relation=%p', relation_p
 577         jmp .next_relation
 578 align_nops
 579 .have_more_members:
 580         cmp al, 0x00
 581         cmove idx_slot_p, nodes_idx
 582                 je .idx_selected
 583         cmp al, 0x01
 584         cmove idx_slot_p, ways_idx
 585                 je .idx_selected
 586         cmp al, 0x02
 587         cmove idx_slot_p, relations_idx
 588 .idx_selected:
 589         xor bit_val_select, bit_val_select ; bit_val_select = 0
 590         mov id_bit_idx_msb, -1 ; this is to make it work with ref = 0
 591         bsr id_bit_idx_msb, id ; if id = 0, ref_bit_idx_msb is untouched namely -1
 592         inc id_bit_idx_msb_d ; store the idx + 1, 32bits because of the following jecxz
 593 align_nops
 594 .next_id_bit:
 595         jecxz .idx_slot_lookup_done ; bit_idx + 1 is stored in cx then 0 means we are finished
 596         bt id, 0
 597         setc bit_val_select_b
 598         mov idx_slot_p, [idx_slot_p + 8 * bit_val_select]
 599         test idx_slot_p, idx_slot_p ; if the idx slot in 0, we have a missing node
 600                 jz .missing_node ; unlikely
 601 if ~SYSTEM_LEGACY
 602         prefetcht0 [idx_slot_p]
 603 end if
 604         shr id, 1
 605         dec id_bit_idx_msb_d
 606         jmp .next_id_bit
 607 align_nops
 608 .idx_slot_lookup_done:
 609         mov rax, [idx_slot_p + 2 * 8] ; XXX: all 3 idxs (nodes/ways/relations) have the same struct
 610         mov [member_p + array.relation_t.member_t.p], rax
 611         TRACE_2ND_PASS 'id found in idx, replacing with pointer %p', rax
 612 .skip_member_role: ; from unlikely code code below
 613         add member_p, array.relation_t.member_t.role + 1 ; points on the byte right after the byte containing the sz of the role str in bytes
 614         movzx rax, byte [member_p - 1]
 615         add member_p, rax
 616         jmp .next_member
 617
 618 .missing_node: ; unlikely
 619         mov qword [member_p + array.relation_t.member_t.p], 0
 620         TRACE_2ND_PASS 'missing id, pointer to 0'
 621         jmp .skip_member_role
 622 purge idx_slot_p
 623 purge id_bit_idx_msb_d
 624 purge id_bit_idx_msb
 625 purge bit_val_select_b
 626 purge bit_val_select
 627 purge id
 628 purge relations_idx
 629 purge ways_idx
 630 purge nodes_idx
 631 purge member_p
 632 purge relations_finish_p
 633 purge relation_p
 634 purge flags
 635 ; }}} resolve_ids ----------------------------------------------------------------------------------
 636 ;===================================================================================================
 637 purge memcpy_link
 638 purge memcpy_bytes_n
 639 purge memcpy_src
 640 purge memcpy_dst
 641 ;{{{ macros ----------------------------------------------------------------------------------------
 642 if TRACE_RELATIONS_IDX_ENABLED
 643         macro TRACE_RELATIONS_IDX fmt, regs&
 644                 TRACE_PREFIX 'RELATIONS_IDX', fmt, regs
 645         end macro
 646 else
 647         macro TRACE_RELATIONS_IDX fmt, regs&
 648         end macro
 649 end if
 650 ;---------------------------------------------------------------------------------------------------
 651 if TRACE_RELATIONS_2ND_PASS_ENABLED
 652         macro TRACE_2ND_PASS fmt, regs&
 653                 TRACE_PREFIX 'RELATIONS_2ND_PASS', fmt, regs
 654         end macro
 655 else
 656         macro TRACE_2ND_PASS fmt, regs&
 657         end macro
 658 end if
 659 ;}}} macros ----------------------------------------------------------------------------------------
 660 end namespace ; relations