Hacking my Pedometer++ data

#!/usr/bin/env ruby
# frozen_string_literal: true

require "bundler/inline"
require "time"
require "set"
require "open3"
require "json"

gemfile do
  source "https://rubygems.org"
  gem "nokogiri"
end

# <Record
#   type="HKQuantityTypeIdentifierFlightsClimbed"
#   sourceName="Mercury"
#   unit="count"
#   creationDate="2014-11-28 18:03:26 +0200"
#   startDate="2014-11-28 17:54:52 +0200"
#   endDate="2014-11-28 17:54:54 +0200"
#   value="2" />

class FloorsParser < Nokogiri::XML::SAX::Document
  HKQuantityTypeIdentifierFlightsClimbed = "HKQuantityTypeIdentifierFlightsClimbed"

  OUTPUT_COUNT = 10_000

  attr_reader :start_date_filter, :end_date_filter, :start_time_filter,
    :end_time_filter, :allowed_sources, :found_sources, :data

  def initialize(start_date:, end_date:, allowed_sources: [])
    @start_date_filter = start_date
    @end_date_filter = end_date
    @start_time_filter = start_date.to_time.freeze
    @end_time_filter = Time.new(
      end_date.year, end_date.month, end_date.day,
      23, 59, 59
    ).freeze
    @allowed_sources = allowed_sources
  end

  def start_document
    puts "Starting document processing... (each dot represents #{OUTPUT_COUNT} records)"
    @found_sources = Set.new
    @data ||= {}
    @processed = 0
  end

  def start_element(name, attrs = [])
    case name
    when "Record"
      @record = attrs.to_h

      @processed += 1
      print "." if (@processed % OUTPUT_COUNT) == 0
    end
  end

  def end_element(name)
    case name
    when "Record"
      if @record && @record.fetch("type") == HKQuantityTypeIdentifierFlightsClimbed
        @found_sources.add(@record["sourceName"])

        if allowed_sources.include?(@record["sourceName"])
          start_time = Time.parse(@record["startDate"])
          end_time = Time.parse(@record["endDate"])

          return unless start_time >= start_time_filter && end_time <= end_time_filter

          date = start_time.to_date

          if date != end_time.to_date
            puts "\nDay crossing record! #{date.iso8601}"
            puts "    (#{(end_time - start_time).to_i}s) [#{@record["sourceName"]}]\n"
          end

          @data[date] ||= {}
          @data[date][@record["sourceName"]] ||= 0
          @data[date][@record["sourceName"]] += @record["value"].to_i
        end
      end
      @record = nil
    end
  end

  def end_document
    puts "\n---\n"
    puts "Date range: #{start_date_filter.iso8601} → #{end_date_filter.iso8601}"
    puts "Found sources: #{@found_sources.to_a.join(", ")}"
    puts "Allowed sources: #{allowed_sources.join(", ")}"
    puts "Data count: #{@data.keys.length}"
  end
end

# Create our parser
floors_parser = FloorsParser.new(
  start_date: Date.new(2014, 1, 1),
  end_date: Date.new(2015, 12, 31),
  allowed_sources: ["Mercury"]
)
parser = Nokogiri::XML::SAX::Parser.new(floors_parser)

# Send some XML to the parser
parser.parse(File.open("./export.xml"))

data = floors_parser.data

puts "Decompressing steps data"
decoded, stderr, status = Open3.capture3("compression_tool -decode -i ./Export.steps")

if status.success?
  steps = JSON.parse(decoded)

  puts "Fixing floors ascended data"
  steps["StepCount"].each do |item|
    # Parse the Core Data timestamp
    time = Time.strptime((item["intervalStart"] + 978307200).to_s, "%s")
    # item["intervalStart"] = time # for debugging

    if (flights = data.dig(time.to_date, "Mercury"))
      item["floorsAscended"] = flights
    else
      item["floorsAscended"] = nil
    end
  end

  puts "Replacing missing floors ascended data with the daily average"
  values = steps["StepCount"].map{|i| i["floorsAscended"]}.compact
  average = (values.sum.to_f / values.length).round
  steps["StepCount"].each do |item|
    item["floorsAscended"] = average if item["floorsAscended"].nil?
  end

  puts "Saving the fixed data"
  filename = "Export-fixed-#{Time.now.to_i}.steps"

  Open3.popen3("compression_tool -encode -o ./#{filename}") { |stdin, stdout, stderr, wait_thr|
    stdin.write JSON.pretty_generate(steps, space_before: " ")
    stdin.close

    exit_status = wait_thr.value

    if exit_status.success?
      puts "Success! Generated #{filename}"
    else
      exit exit_status.to_i
    end
  }
else
  puts "Decoding steps failed"
  puts stderr
  exit status.to_i
end