Не удалять активный класс по скроллу

const wrapper = document.querySelector('.wrapper');
const box = wrapper.querySelectorAll('.box');

document.addEventListener('scroll', () => {
  const scroll = document.documentElement.scrollTop;
  
  box.forEach((item, i) => {
    const offsetTopText = box[i].getBoundingClientRect().top + pageYOffset;
    const offsetHeightText = box[i].getBoundingClientRect().height;
    
    if (scroll > offsetTopText && scroll < (offsetTopText + offsetHeightText)) {
      box[i].classList.add('active');
    } else {
      box[i].classList.remove('active');
    }
  });
});

.wrapper {
  display: flex;
  flex-direction: column;
}

.box {
  padding: 50px 0;
  width: 250px;
  height: 150px;
  margin-bottom: 10px;
  background: green;
}

.large {
  width: 250px;
  height: 170px;
  background: gold;
}

.active {
  background: red;
}

<div class="wrapper">
  <div class="large">large</div>
  <div class="large">large</div>
  <div class="large">large</div>
  <div class="box active">box 1</div>
  <div class="box">box 2</div>
  <div class="box">box 3</div>
  <div class="large">large</div>
  <div class="large">large</div>
  <div class="large">large</div>
</div>