diff --git a/mhkit/dolfyn/io/nortek2.py b/mhkit/dolfyn/io/nortek2.py index a536bca5..3ec560f5 100644 --- a/mhkit/dolfyn/io/nortek2.py +++ b/mhkit/dolfyn/io/nortek2.py @@ -301,9 +301,11 @@ def _read_str(self, size): string = string[1:-1] return id, string - def _read_burst(self, id, dat, c, echo=False): + def _read_burst(self, id, dat, c, sz, echo=False): rdr = self._burst_readers[id] - rdr.read_into(self.f, dat, c) + to_skip = rdr.read_into(self.f, dat, c, sz) + if to_skip: + self.f.seek(to_skip, 1) def readfile(self, ens_start=0, ens_stop=None): # If the lastblock is not whole, we don't read it. @@ -326,10 +328,13 @@ def readfile(self, ens_start=0, ens_stop=None): except IOError: return outdat id = hdr["id"] - if id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast), + if id not in outdat: + # If the ID was skipped in the indexer, it won't be in outdat + self.f.seek(hdr["sz"], 1) + elif id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast), # "avg data record" (vel_avg + ast_avg), "bottom track data record" (bt), # "interleaved burst data record" (vel_b5), "echosounder record" (echo) - self._read_burst(id, outdat[id], c) + self._read_burst(id, outdat[id], c, hdr["sz"]) elif id in [26, 31]: # "burst altimeter raw record" (_altraw), "avg altimeter raw record" (_altraw_avg) rdr = self._burst_readers[id] @@ -367,7 +372,7 @@ def readfile(self, ens_start=0, ens_stop=None): "The number of samples in this 'Altimeter Raw' " "burst is different from prior bursts." ) - self._read_burst(id, outdat[id], c_altraw[id]) + self._read_burst(id, outdat[id], c_altraw[id], hdr["sz"]) outdat[id]["ensemble"][c_altraw[id]] = c c_altraw[id] += 1 @@ -520,7 +525,8 @@ def _reorg(dat): dnow["standard_name"].pop(ky) outdat["standard_name"].update(dnow["standard_name"]) cfg["burst_config" + tag] = lib._headconfig_int2dict( - lib._collapse(dnow["config"], exclude=collapse_exclude, name="config") + lib._collapse(dnow["config"], exclude=collapse_exclude, name="config"), + mode="bt" if "bt" in tag else "burst", ) outdat["coords"]["time" + tag] = lib._calc_time( dnow["year"] + np.uint16(1900), diff --git a/mhkit/dolfyn/io/nortek2_defs.py b/mhkit/dolfyn/io/nortek2_defs.py index 56cc0e05..f30b0b7d 100644 --- a/mhkit/dolfyn/io/nortek2_defs.py +++ b/mhkit/dolfyn/io/nortek2_defs.py @@ -71,13 +71,14 @@ def init_data(self, npings): out[nm] = _nans(shp + [npings], dtype=np.dtype(fmt[0])) return out - def read_into(self, fobj, data, ens, cs=None): - dat_tuple = self.read(fobj, cs=cs) + def read_into(self, fobj, data, ens, sz, cs=None): + dat_tuple, to_skip = self.read(fobj, sz=sz, cs=cs) for nm, shp, d in zip(self._names, self._shape, dat_tuple): try: data[nm][..., ens] = d except ValueError: data[nm][..., ens] = np.asarray(d).reshape(shp) + return to_skip @property def format( @@ -85,8 +86,11 @@ def format( ): return _format(self._format, self._N) - def read(self, fobj, cs=None): + def read(self, fobj, sz=None, cs=None): bytes = fobj.read(self.nbyte) + # For newer versions of id data packets + if sz is not None: + extra_bytes = sz - self.nbyte if len(bytes) != self.nbyte: raise IOError("End of file.") data = self._struct.unpack(bytes) @@ -110,7 +114,10 @@ def read(self, fobj, cs=None): else: out.append(data[c : (c + n)]) c += n - return out + if sz is not None: + return out, extra_bytes + else: + return out def read2dict(self, fobj, cs=False): return {self._names[idx]: dat for idx, dat in enumerate(self.read(fobj, cs=cs))} diff --git a/mhkit/dolfyn/io/nortek2_lib.py b/mhkit/dolfyn/io/nortek2_lib.py index 7c1f17ee..92b7c499 100644 --- a/mhkit/dolfyn/io/nortek2_lib.py +++ b/mhkit/dolfyn/io/nortek2_lib.py @@ -176,7 +176,7 @@ def _create_index(infile, outfile, init_pos, eof, debug): if init_pos and not pos: fin.seek(init_pos, 1) try: - dat = struct.unpack(" 0: if (ens[idk] == 1) or (ens[idk] < last_ens[idk]): @@ -232,7 +245,7 @@ def _create_index(infile, outfile, init_pos, eof, debug): # Covers ID keys not saved in sequential order ens[idk] = last_ens[idk] + 1 - if last_ens[idk] > 0 and last_ens[idk] != ens[idk]: + if last_ens[idk] > 0 and (last_ens[idk] != ens[idk]): N[idk] += 1 fout.write( @@ -262,7 +275,7 @@ def _create_index(infile, outfile, init_pos, eof, debug): # File Position: Valid ID keys (1A, 10), Hex ID, Length in bytes, Ensemble #, Last Ensemble Found' # hex: [18, 15, 1C, 17] = [vel_b5, vel, echo, bt] logging.info( - "%10d: %02X, %d, %02X, %d, %d, %d, %d\n" + "%10d: %02X, %d, %02X, %d, %d, %d, %d" % ( pos, dat[0], @@ -309,8 +322,17 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False): for id in uid: # These are the indices for this ID inds = np.nonzero(idx["ID"] == id)[0] + # Sometimes there are multiple id pings in a single ping cycle + diff = np.diff(inds) + if len(set(diff)) == 1: + N_id = diff[0] + # If an id has multiple pings/cycle but also skipped pings, + # the id carrying multiple pings will have a smaller "diff" + # than the id's that do not. Therefore any skipped pings + # will have a "diff" greater than the original "N_id" + # These are bad steps in the indices for this ID - ibad = np.nonzero(np.diff(inds) > N_id)[0] + ibad = np.nonzero(diff > N_id)[0] # Check if spacing is equal for dual profiling ADCPs if dp: @@ -328,7 +350,7 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False): # the ensemble count is wrong. logging.warning( "Skipped ping (ID: {}) in file {} at ensemble {}.".format( - id, infile, idx["ens"][inds[ib + 1] - 1] + hex(id), infile, idx["ens"][inds[ib + 1] - 1] ) ) hwe[inds[(ib + 1) :]] += 1 @@ -509,7 +531,8 @@ def _headconfig_int2dict(val, mode="burst"): tilt_valid=_getbit(val, 3), # bit 4 is unused vel=_getbit(val, 5), - # bits 6-7 unused + amp=_getbit(val, 6), + corr=_getbit(val, 7), dist=_getbit(val, 8), fom=_getbit(val, 9), ahrs=_getbit(val, 10),