Skip to content

Nortek Bottom Track #471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jmcvey3 wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Nortek Bottom Track #471

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions mhkit/dolfyn/io/nortek2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,9 +301,11 @@ def _read_str(self, size):
string = string[1:-1]
return id, string

def _read_burst(self, id, dat, c, echo=False):
def _read_burst(self, id, dat, c, sz, echo=False):
rdr = self._burst_readers[id]
rdr.read_into(self.f, dat, c)
to_skip = rdr.read_into(self.f, dat, c, sz)
if to_skip:
self.f.seek(to_skip, 1)

def readfile(self, ens_start=0, ens_stop=None):
# If the lastblock is not whole, we don't read it.
Expand All @@ -326,10 +328,13 @@ def readfile(self, ens_start=0, ens_stop=None):
except IOError:
return outdat
id = hdr["id"]
if id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast),
if id not in outdat:
# If the ID was skipped in the indexer, it won't be in outdat
self.f.seek(hdr["sz"], 1)
elif id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast),
# "avg data record" (vel_avg + ast_avg), "bottom track data record" (bt),
# "interleaved burst data record" (vel_b5), "echosounder record" (echo)
self._read_burst(id, outdat[id], c)
self._read_burst(id, outdat[id], c, hdr["sz"])
elif id in [26, 31]:
# "burst altimeter raw record" (_altraw), "avg altimeter raw record" (_altraw_avg)
rdr = self._burst_readers[id]
Expand Down Expand Up @@ -367,7 +372,7 @@ def readfile(self, ens_start=0, ens_stop=None):
"The number of samples in this 'Altimeter Raw' "
"burst is different from prior bursts."
)
self._read_burst(id, outdat[id], c_altraw[id])
self._read_burst(id, outdat[id], c_altraw[id], hdr["sz"])
outdat[id]["ensemble"][c_altraw[id]] = c
c_altraw[id] += 1

Expand Down Expand Up @@ -520,7 +525,8 @@ def _reorg(dat):
dnow["standard_name"].pop(ky)
outdat["standard_name"].update(dnow["standard_name"])
cfg["burst_config" + tag] = lib._headconfig_int2dict(
lib._collapse(dnow["config"], exclude=collapse_exclude, name="config")
lib._collapse(dnow["config"], exclude=collapse_exclude, name="config"),
mode="bt" if "bt" in tag else "burst",
)
outdat["coords"]["time" + tag] = lib._calc_time(
dnow["year"] + np.uint16(1900),
Expand Down
15 changes: 11 additions & 4 deletions mhkit/dolfyn/io/nortek2_defs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,22 +71,26 @@ def init_data(self, npings):
out[nm] = _nans(shp + [npings], dtype=np.dtype(fmt[0]))
return out

def read_into(self, fobj, data, ens, cs=None):
dat_tuple = self.read(fobj, cs=cs)
def read_into(self, fobj, data, ens, sz, cs=None):
dat_tuple, to_skip = self.read(fobj, sz=sz, cs=cs)
for nm, shp, d in zip(self._names, self._shape, dat_tuple):
try:
data[nm][..., ens] = d
except ValueError:
data[nm][..., ens] = np.asarray(d).reshape(shp)
return to_skip

@property
def format(
self,
):
return _format(self._format, self._N)

def read(self, fobj, cs=None):
def read(self, fobj, sz=None, cs=None):
bytes = fobj.read(self.nbyte)
# For newer versions of id data packets
if sz is not None:
extra_bytes = sz - self.nbyte
if len(bytes) != self.nbyte:
raise IOError("End of file.")
data = self._struct.unpack(bytes)
Expand All @@ -110,7 +114,10 @@ def read(self, fobj, cs=None):
else:
out.append(data[c : (c + n)])
c += n
return out
if sz is not None:
return out, extra_bytes
else:
return out

def read2dict(self, fobj, cs=False):
return {self._names[idx]: dat for idx, dat in enumerate(self.read(fobj, cs=cs))}
Expand Down
45 changes: 34 additions & 11 deletions mhkit/dolfyn/io/nortek2_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def _create_index(infile, outfile, init_pos, eof, debug):
if init_pos and not pos:
fin.seek(init_pos, 1)
try:
dat = struct.unpack("<BBBBhhh", fin.read(10))
dat = struct.unpack("<BBBBHHH", fin.read(10))
except Exception as e:
logging.error(e)
logging.error("Header is shorter than 10 bytes: %10d\n" % (pos))
Expand All @@ -187,7 +187,7 @@ def _create_index(infile, outfile, init_pos, eof, debug):
if dat[1] != 10:
if dat[1] == 12:
fin.seek(-10, 1)
dat = struct.unpack("<BBBBIhh", fin.read(12))
dat = struct.unpack("<BBBBIHH", fin.read(12))
else:
# Lost header
if debug:
Expand All @@ -213,26 +213,39 @@ def _create_index(infile, outfile, init_pos, eof, debug):
if dat[2] in ids:
idk = dat[2]
# version, byte offset to actual data, configuration bit mask
pos_common_data = fin.tell()
d_ver, d_off, config = struct.unpack("<BBH", fin.read(4))
if d_ver not in [1, 3, 7]:
# data-record definition version number
# Data record definition version number
# increases occasionally with instrument firmware updates
continue
logging.debug(
f"Unknown data record version number {d_ver} for id {hex(idk)}"
)
# Not sure how to handle data record definition 9 for BT
if d_ver != 9:
fin.seek(d_off - 4 + dat[4] - (36 + seek_2ens[idk]), 1)
pos = fin.tell()
last_ens[idk] = ens[idk]
continue

fin.seek(4, 1) # instrument serial number
yr, mo, dy, h, m, s, u = struct.unpack("6BH", fin.read(8))
fin.seek(14, 1) # soundspeed, temp, pres, h,p,r
beams_cy = struct.unpack("<H", fin.read(2))[0] # cell_size
# cell_size (either in bytes or mm?)
beams_cy = struct.unpack("<H", fin.read(2))[0]
fin.seek(seek_2ens[dat[2]], 1)
# should end at byte 74 or 76
ens[idk] = struct.unpack("<I", fin.read(4))[0]
# we should now be "d_off" bytes later
if fin.tell() - pos_common_data != d_off:
logging.warning(f" {fin.tell()} Data offset incorrect!")

if last_ens[idk] > 0:
if (ens[idk] == 1) or (ens[idk] < last_ens[idk]):
# Covers all id keys saved in "burst mode"
# Covers ID keys not saved in sequential order
ens[idk] = last_ens[idk] + 1

if last_ens[idk] > 0 and last_ens[idk] != ens[idk]:
if last_ens[idk] > 0 and (last_ens[idk] != ens[idk]):
N[idk] += 1

fout.write(
Expand Down Expand Up @@ -262,7 +275,7 @@ def _create_index(infile, outfile, init_pos, eof, debug):
# File Position: Valid ID keys (1A, 10), Hex ID, Length in bytes, Ensemble #, Last Ensemble Found'
# hex: [18, 15, 1C, 17] = [vel_b5, vel, echo, bt]
logging.info(
"%10d: %02X, %d, %02X, %d, %d, %d, %d\n"
"%10d: %02X, %d, %02X, %d, %d, %d, %d"
% (
pos,
dat[0],
Expand Down Expand Up @@ -309,8 +322,17 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False):
for id in uid:
# These are the indices for this ID
inds = np.nonzero(idx["ID"] == id)[0]
# Sometimes there are multiple id pings in a single ping cycle
diff = np.diff(inds)
if len(set(diff)) == 1:
N_id = diff[0]
# If an id has multiple pings/cycle but also skipped pings,
# the id carrying multiple pings will have a smaller "diff"
# than the id's that do not. Therefore any skipped pings
# will have a "diff" greater than the original "N_id"

# These are bad steps in the indices for this ID
ibad = np.nonzero(np.diff(inds) > N_id)[0]
ibad = np.nonzero(diff > N_id)[0]

# Check if spacing is equal for dual profiling ADCPs
if dp:
Expand All @@ -328,7 +350,7 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False):
# the ensemble count is wrong.
logging.warning(
"Skipped ping (ID: {}) in file {} at ensemble {}.".format(
id, infile, idx["ens"][inds[ib + 1] - 1]
hex(id), infile, idx["ens"][inds[ib + 1] - 1]
)
)
hwe[inds[(ib + 1) :]] += 1
Expand Down Expand Up @@ -509,7 +531,8 @@ def _headconfig_int2dict(val, mode="burst"):
tilt_valid=_getbit(val, 3),
# bit 4 is unused
vel=_getbit(val, 5),
# bits 6-7 unused
amp=_getbit(val, 6),
corr=_getbit(val, 7),
dist=_getbit(val, 8),
fom=_getbit(val, 9),
ahrs=_getbit(val, 10),
Expand Down
Loading