REFACTOR: packages + OfferRepository

This commit is contained in:
Patrick Haßel 2024-06-07 09:53:32 +02:00
parent fcc6403dbf
commit 4caea1a80e
13 changed files with 211 additions and 188 deletions

View File

@ -67,7 +67,7 @@
<configuration> <configuration>
<archive> <archive>
<manifest> <manifest>
<mainClass>de.ph87.kleinanzeigen.api.Main</mainClass> <mainClass>de.ph87.kleinanzeigen.Main</mainClass>
</manifest> </manifest>
</archive> </archive>
</configuration> </configuration>

View File

@ -1,4 +1,4 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;

View File

@ -0,0 +1,48 @@
package de.ph87.kleinanzeigen;
import de.ph87.kleinanzeigen.kleinanzeigen.KleinanzeigenApi;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository;
import de.ph87.kleinanzeigen.telegram.TelegramBot;
import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import java.io.IOException;
import java.util.List;
@Slf4j
@SuppressWarnings({"InfiniteLoopStatement", "SameParameterValue", "SynchronizationOnLocalVariableOrMethodParameter"})
public class Main {
private static TelegramBot telegramBot;
private static final OfferRepository offerRepository = new OfferRepository(offer -> telegramBot.remove(List.of(offer)));
private static final KleinanzeigenApi kleinanzeigenApi = new KleinanzeigenApi(offerRepository);
public static void main(String[] args) throws IOException, TelegramApiException {
telegramBot = new TelegramBot(offerRepository);
try {
while (true) {
handle(telegramBot);
waitSeconds(60);
}
} catch (InterruptedException e) {
log.warn(e.toString());
} finally {
telegramBot.stop();
}
}
private static void handle(final TelegramBot telegramBot) {
kleinanzeigenApi.fetchUntilDuplicate(5);
offerRepository.findAll().stream().filter(offer -> offer.getTelegramMessageId() == null).forEach(telegramBot::send);
}
private static void waitSeconds(final long seconds) throws InterruptedException {
final Object lock = new Object();
synchronized (lock) {
lock.wait(seconds * 1000);
}
}
}

View File

@ -1,43 +0,0 @@
package de.ph87.kleinanzeigen.api;
import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import java.io.IOException;
import java.util.List;
@Slf4j
@SuppressWarnings({"InfiniteLoopStatement", "SameParameterValue", "SynchronizationOnLocalVariableOrMethodParameter"})
public class Main {
private static Bot bot;
private static final Kleinanzeigen kleinanzeigen = new Kleinanzeigen(offer -> bot.remove(List.of(offer)));
public static void main(String[] args) throws IOException, TelegramApiException {
bot = new Bot(kleinanzeigen::ignore, kleinanzeigen::findByTelegramMessageId, kleinanzeigen::remember);
try {
while (true) {
handle(bot);
waitSeconds(60);
}
} catch (InterruptedException e) {
log.warn(e.toString());
} finally {
bot.stop();
}
}
private static void handle(final Bot bot) {
kleinanzeigen.fetchUntilDuplicate(5);
kleinanzeigen.findAll().stream().filter(offer -> offer.getTelegramMessageId() == null).forEach(bot::send);
}
private static void waitSeconds(final long seconds) throws InterruptedException {
final Object lock = new Object();
synchronized (lock) {
lock.wait(seconds * 1000);
}
}
}

View File

@ -1,4 +1,4 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.kleinanzeigen;
import lombok.Data; import lombok.Data;

View File

@ -1,86 +1,32 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.kleinanzeigen;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.Offer;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferParseException;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.telegram.telegrambots.meta.api.objects.MaybeInaccessibleMessage;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI; import java.net.URI;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalTime; import java.time.LocalTime;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.util.*; import java.util.TimeZone;
import java.util.function.Consumer;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static de.ph87.kleinanzeigen.api.JSON.objectMapper;
@Slf4j @Slf4j
public class Kleinanzeigen { public class KleinanzeigenApi {
private static final int KEEP_LAST_OFFERS_COUNT = 200;
private static final File FILE = new File("./offers.json");
private static final String VERSCHENKEN_EPPELBORN_30KM = "https://www.kleinanzeigen.de/s-zu-verschenken/66571/seite:%d/c192l339r30"; private static final String VERSCHENKEN_EPPELBORN_30KM = "https://www.kleinanzeigen.de/s-zu-verschenken/66571/seite:%d/c192l339r30";
private final List<Offer> offers; private final OfferRepository offerRepository;
private final Consumer<Offer> remove; public KleinanzeigenApi(final OfferRepository offerRepository) {
this.offerRepository = offerRepository;
public Kleinanzeigen(final Consumer<Offer> remove) {
this.remove = remove;
offers = load();
}
private List<Offer> load() {
try {
final List<Offer> offers = objectMapper.readerForListOf(Offer.class).readValue(FILE);
log.info("Loaded {} offers from file: {}", offers.size(), FILE);
return offers;
} catch (IOException e) {
log.warn("Failed to load Offers from file={}: {}", FILE, e.toString());
return new ArrayList<>();
}
}
private void save() {
try {
final List<Offer> removed;
synchronized (offers) {
removed = _cleanUp();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(FILE, offers);
log.debug("Wrote {} offers to file: {}", offers.size(), FILE);
}
removed.forEach(remove);
} catch (IOException e) {
log.warn("Failed to write Offers to file={}: {}", FILE, e.toString());
}
}
private List<Offer> _cleanUp() {
if (offers.stream().anyMatch(Offer::_deleted_)) {
throw new RuntimeException();
}
offers.sort(Comparator.comparing(Offer::getDate));
final ZonedDateTime now = ZonedDateTime.now();
final List<Offer> deleted = new ArrayList<>();
final List<Offer> removable = new ArrayList<>(offers.stream().filter(offer -> !offer.isRemember() && (offer.getRememberUntil() == null || now.isAfter(offer.getRememberUntil()))).toList());
while (!removable.isEmpty() && removable.size() > Kleinanzeigen.KEEP_LAST_OFFERS_COUNT) {
final Offer offer = removable.removeFirst();
offers.remove(offer);
offer.markDeleted();
deleted.add(offer);
}
return deleted;
} }
public void fetchUntilDuplicate(final int maxPageCount) { public void fetchUntilDuplicate(final int maxPageCount) {
@ -108,10 +54,10 @@ public class Kleinanzeigen {
fetchResult.add(MergeResult.ERROR); fetchResult.add(MergeResult.ERROR);
continue; continue;
} }
final MergeResult mergeResult = merge(offer); final MergeResult mergeResult = offerRepository.save(offer);
fetchResult.add(mergeResult); fetchResult.add(mergeResult);
} }
save(); offerRepository.flush();
} catch (IOException e) { } catch (IOException e) {
log.error("Failed to fetch Kleinanzeigen: {}", e.toString()); log.error("Failed to fetch Kleinanzeigen: {}", e.toString());
} }
@ -165,20 +111,6 @@ public class Kleinanzeigen {
return ""; return "";
} }
private MergeResult merge(final Offer offer) {
synchronized (offer) {
final Optional<Offer> existingOptional = offers.stream().filter(existing -> existing.getId().equals(offer.getId())).findFirst();
if (existingOptional.isPresent()) {
existingOptional.get().merge(offer);
return MergeResult.UPDATED;
} else {
log.info("Created: {}", offer);
offers.add(offer);
return MergeResult.CREATED;
}
}
}
private ZonedDateTime parseDate(final String text) { private ZonedDateTime parseDate(final String text) {
final Matcher dayNameMatcher = Pattern.compile("(?<day>Gestern|Heute), (?<hour>\\d+):(?<minute>\\d+)").matcher(text); final Matcher dayNameMatcher = Pattern.compile("(?<day>Gestern|Heute), (?<hour>\\d+):(?<minute>\\d+)").matcher(text);
if (dayNameMatcher.find()) { if (dayNameMatcher.find()) {
@ -193,36 +125,4 @@ public class Kleinanzeigen {
throw new NumberFormatException("Failed to parse date: " + text); throw new NumberFormatException("Failed to parse date: " + text);
} }
public List<Offer> findAll() {
synchronized (offers) {
return new ArrayList<>(offers);
}
}
public void ignore(final MaybeInaccessibleMessage message) {
synchronized (offers) {
findByTelegramMessageId(message).ifPresent(offer -> {
offer.ignore();
save();
});
}
}
public Optional<Offer> remember(final MaybeInaccessibleMessage message, final boolean remember) {
synchronized (offers) {
final Optional<Offer> optional = findByTelegramMessageId(message);
optional.ifPresent(offer -> {
offer.setRemember(remember);
save();
});
return optional;
}
}
public Optional<Offer> findByTelegramMessageId(final MaybeInaccessibleMessage message) {
synchronized (offers) {
return offers.stream().filter(offer -> Objects.equals(offer.getTelegramMessageId(), message.getMessageId())).findFirst();
}
}
} }

View File

@ -1,4 +1,4 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.kleinanzeigen;
public enum MergeResult { public enum MergeResult {
CREATED, UPDATED, ERROR CREATED, UPDATED, ERROR

View File

@ -1,4 +1,4 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;

View File

@ -1,4 +1,4 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;

View File

@ -0,0 +1,122 @@
package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import de.ph87.kleinanzeigen.kleinanzeigen.MergeResult;
import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.meta.api.objects.MaybeInaccessibleMessage;
import java.io.File;
import java.io.IOException;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.function.Consumer;
import static de.ph87.kleinanzeigen.JSON.objectMapper;
@Slf4j
public class OfferRepository {
private static final int KEEP_LAST_OFFERS_COUNT = 200;
private static final File FILE = new File("./offers.json");
private final List<Offer> offers;
private final Consumer<Offer> remove;
public OfferRepository(final Consumer<Offer> remove) {
this.remove = remove;
offers = load();
}
private List<Offer> load() {
try {
final List<Offer> offers = objectMapper.readerForListOf(Offer.class).readValue(FILE);
log.info("Loaded {} offers from file: {}", offers.size(), FILE);
return offers;
} catch (IOException e) {
log.warn("Failed to load Offers from file={}: {}", FILE, e.toString());
return new ArrayList<>();
}
}
public void flush() {
try {
final List<Offer> removed;
synchronized (offers) {
removed = _cleanUp();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(FILE, offers);
log.debug("Wrote {} offers to file: {}", offers.size(), FILE);
}
removed.forEach(remove);
} catch (IOException e) {
log.warn("Failed to write Offers to file={}: {}", FILE, e.toString());
}
}
private List<Offer> _cleanUp() {
if (offers.stream().anyMatch(Offer::_deleted_)) {
throw new RuntimeException();
}
offers.sort(Comparator.comparing(Offer::getDate));
final ZonedDateTime now = ZonedDateTime.now();
final List<Offer> deleted = new ArrayList<>();
final List<Offer> removable = new ArrayList<>(offers.stream().filter(offer -> !offer.isRemember() && (offer.getRememberUntil() == null || now.isAfter(offer.getRememberUntil()))).toList());
while (!removable.isEmpty() && removable.size() > OfferRepository.KEEP_LAST_OFFERS_COUNT) {
final Offer offer = removable.removeFirst();
offers.remove(offer);
offer.markDeleted();
deleted.add(offer);
}
return deleted;
}
public MergeResult save(final Offer offer) {
synchronized (offer) {
final Optional<Offer> existingOptional = offers.stream().filter(existing -> existing.getId().equals(offer.getId())).findFirst();
if (existingOptional.isPresent()) {
existingOptional.get().merge(offer);
return MergeResult.UPDATED;
} else {
log.info("Created: {}", offer);
offers.add(offer);
return MergeResult.CREATED;
}
}
}
public List<Offer> findAll() {
synchronized (offers) {
return new ArrayList<>(offers);
}
}
public void ignore(final MaybeInaccessibleMessage message) {
synchronized (offers) {
findByTelegramMessageId(message).ifPresent(offer -> {
offer.ignore();
flush();
});
}
}
public Optional<Offer> remember(final MaybeInaccessibleMessage message, final boolean remember) {
synchronized (offers) {
final Optional<Offer> optional = findByTelegramMessageId(message);
optional.ifPresent(offer -> {
offer.setRemember(remember);
flush();
});
return optional;
}
}
public Optional<Offer> findByTelegramMessageId(final MaybeInaccessibleMessage message) {
synchronized (offers) {
return offers.stream().filter(offer -> Objects.equals(offer.getTelegramMessageId(), message.getMessageId())).findFirst();
}
}
}

View File

@ -1,4 +1,4 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.telegram;
public enum InlineCommand { public enum InlineCommand {
IGNORE, REMEMBER, UNREMEMBER IGNORE, REMEMBER, UNREMEMBER

View File

@ -1,4 +1,4 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.telegram;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data; import lombok.Data;

View File

@ -1,6 +1,8 @@
package de.ph87.kleinanzeigen.api; package de.ph87.kleinanzeigen.telegram;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.Offer;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.bots.TelegramLongPollingBot; import org.telegram.telegrambots.bots.TelegramLongPollingBot;
import org.telegram.telegrambots.meta.TelegramBotsApi; import org.telegram.telegrambots.meta.TelegramBotsApi;
@ -20,15 +22,15 @@ import java.io.ByteArrayOutputStream;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.ArrayList;
import java.util.function.BiFunction; import java.util.Arrays;
import java.util.function.Consumer; import java.util.List;
import java.util.function.Function; import java.util.Objects;
import static de.ph87.kleinanzeigen.api.JSON.objectMapper; import static de.ph87.kleinanzeigen.JSON.objectMapper;
@Slf4j @Slf4j
public class Bot extends TelegramLongPollingBot { public class TelegramBot extends TelegramLongPollingBot {
private static final long CHAT_ID = 101138682L; private static final long CHAT_ID = 101138682L;
@ -40,29 +42,29 @@ public class Bot extends TelegramLongPollingBot {
private final DefaultBotSession session; private final DefaultBotSession session;
private final Consumer<MaybeInaccessibleMessage> ignore; private final OfferRepository offerRepository;
private final Function<MaybeInaccessibleMessage, Optional<Offer>> find; public TelegramBot(final OfferRepository offerRepository) throws IOException, TelegramApiException {
private final BiFunction<MaybeInaccessibleMessage, Boolean, Optional<Offer>> remember;
public Bot(final Consumer<MaybeInaccessibleMessage> ignore, final Function<MaybeInaccessibleMessage, Optional<Offer>> find, final BiFunction<MaybeInaccessibleMessage, Boolean, Optional<Offer>> remember) throws IOException, TelegramApiException {
super(readToken()); super(readToken());
this.offerRepository = offerRepository;
final BufferedImage img = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB); final BufferedImage img = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB);
final ByteArrayOutputStream stream = new ByteArrayOutputStream(); final ByteArrayOutputStream stream = new ByteArrayOutputStream();
ImageIO.write(img, "PNG", stream); ImageIO.write(img, "PNG", stream);
NO_IMAGE = stream.toByteArray(); NO_IMAGE = stream.toByteArray();
this.ignore = ignore;
this.find = find;
this.remember = remember;
log.info("Starting telegram bot..."); log.info("Starting telegram bot...");
final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class); final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
session = (DefaultBotSession) api.registerBot(this); session = (DefaultBotSession) api.registerBot(this);
log.info("Telegram bot registered."); log.info("Telegram bot registered.");
} }
private static String readToken() throws IOException {
try (final FileInputStream stream = new FileInputStream("./telegram.token")) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim();
}
}
@Override @Override
public String getBotUsername() { public String getBotUsername() {
return "BotKleinanzeigenBot"; return "BotKleinanzeigenBot";
@ -96,26 +98,26 @@ public class Bot extends TelegramLongPollingBot {
} }
private void ignore(final MaybeInaccessibleMessage message) { private void ignore(final MaybeInaccessibleMessage message) {
ignore.accept(message); offerRepository.ignore(message);
remove(message); remove(message);
} }
private void remember(final MaybeInaccessibleMessage message) { private void remember(final MaybeInaccessibleMessage message) {
remember.apply(message, true).ifPresentOrElse( offerRepository.remember(message, true).ifPresentOrElse(
offer -> updateMessage(message, offer), offer -> updateMessage(message, offer),
() -> remove(message) () -> remove(message)
); );
} }
private void unremember(final MaybeInaccessibleMessage message) { private void unremember(final MaybeInaccessibleMessage message) {
remember.apply(message, false).ifPresentOrElse( offerRepository.remember(message, false).ifPresentOrElse(
offer -> updateMessage(message, offer), offer -> updateMessage(message, offer),
() -> remove(message) () -> remove(message)
); );
} }
private void updateMessage(final MaybeInaccessibleMessage message) { private void updateMessage(final MaybeInaccessibleMessage message) {
find.apply(message).ifPresentOrElse( offerRepository.findByTelegramMessageId(message).ifPresentOrElse(
offer -> updateMessage(message, offer), offer -> updateMessage(message, offer),
() -> remove(message) () -> remove(message)
); );
@ -131,12 +133,6 @@ public class Bot extends TelegramLongPollingBot {
} }
} }
private static String readToken() throws IOException {
try (final FileInputStream stream = new FileInputStream("./telegram.token")) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim();
}
}
public void send(final Offer offer) { public void send(final Offer offer) {
try { try {
final InputFile inputFile = offer.getImage().isEmpty() ? new InputFile(new ByteArrayInputStream(NO_IMAGE), "[Kein Bild]") : new InputFile(offer.getImage()); final InputFile inputFile = offer.getImage().isEmpty() ? new InputFile(new ByteArrayInputStream(NO_IMAGE), "[Kein Bild]") : new InputFile(offer.getImage());