diff --git a/.gitignore b/.gitignore index d87248d..541f3d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /offers.json /.idea/ /telegram.token +/*.db target/ !.mvn/wrapper/maven-wrapper.jar diff --git a/application.properties b/application.properties new file mode 100644 index 0000000..ce03717 --- /dev/null +++ b/application.properties @@ -0,0 +1,9 @@ +logging.level.de.ph87=DEBUG +#- +spring.datasource.url=jdbc:h2:./database;AUTO_SERVER=TRUE +spring.datasource.driverClassName=org.h2.Driver +spring.datasource.username=sa +spring.datasource.password=password +spring.jpa.database-platform=org.hibernate.dialect.H2Dialect +#- +#spring.jpa.hibernate.ddl-auto=create diff --git a/pom.xml b/pom.xml index dd7d1c2..fd30797 100644 --- a/pom.xml +++ b/pom.xml @@ -14,65 +14,45 @@ UTF-8 + + org.springframework.boot + spring-boot-starter-parent + 3.3.0 + + + + org.springframework.boot + spring-boot-starter-json + 3.3.0 + + + org.springframework.boot + spring-boot-starter-data-jpa + + + com.h2database + h2 + org.projectlombok lombok - 1.18.32 + + + javax.xml.bind + jaxb-api + 2.3.1 org.jsoup jsoup - 1.16.1 + 1.17.2 org.telegram telegrambots 6.9.7.1 - - org.slf4j - slf4j-simple - 2.0.12 - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - 2.15.4 - - - - - maven-assembly-plugin - 3.6.0 - - - package - - single - - - - - - jar-with-dependencies - - - - - maven-jar-plugin - 3.3.0 - - - - de.ph87.kleinanzeigen.Main - - - - - - - \ No newline at end of file diff --git a/src/main/java/de/ph87/kleinanzeigen/Application.java b/src/main/java/de/ph87/kleinanzeigen/Application.java new file mode 100644 index 0000000..1f728d0 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/Application.java @@ -0,0 +1,17 @@ +package de.ph87.kleinanzeigen; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@Slf4j +@SpringBootApplication +@RequiredArgsConstructor +public class Application { + + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/JSON.java b/src/main/java/de/ph87/kleinanzeigen/JSON.java deleted file mode 100644 index fa21ce5..0000000 --- a/src/main/java/de/ph87/kleinanzeigen/JSON.java +++ /dev/null @@ -1,9 +0,0 @@ -package de.ph87.kleinanzeigen; - -import com.fasterxml.jackson.databind.ObjectMapper; - -public class JSON { - - public static final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); - -} diff --git a/src/main/java/de/ph87/kleinanzeigen/Main.java b/src/main/java/de/ph87/kleinanzeigen/Main.java deleted file mode 100644 index 282a983..0000000 --- a/src/main/java/de/ph87/kleinanzeigen/Main.java +++ /dev/null @@ -1,48 +0,0 @@ -package de.ph87.kleinanzeigen; - -import de.ph87.kleinanzeigen.kleinanzeigen.KleinanzeigenApi; -import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository; -import de.ph87.kleinanzeigen.telegram.TelegramBot; -import lombok.extern.slf4j.Slf4j; -import org.telegram.telegrambots.meta.exceptions.TelegramApiException; - -import java.io.IOException; -import java.util.List; - -@Slf4j -@SuppressWarnings({"InfiniteLoopStatement", "SameParameterValue", "SynchronizationOnLocalVariableOrMethodParameter"}) -public class Main { - - private static TelegramBot telegramBot; - - private static final OfferRepository offerRepository = new OfferRepository(offer -> telegramBot.remove(List.of(offer))); - - private static final KleinanzeigenApi kleinanzeigenApi = new KleinanzeigenApi(offerRepository); - - public static void main(String[] args) throws IOException, TelegramApiException { - telegramBot = new TelegramBot(offerRepository); - try { - while (true) { - handle(telegramBot); - waitSeconds(60); - } - } catch (InterruptedException e) { - log.warn(e.toString()); - } finally { - telegramBot.stop(); - } - } - - private static void handle(final TelegramBot telegramBot) { - kleinanzeigenApi.fetchUntilDuplicate(5); - offerRepository.findAll().stream().filter(offer -> offer.getTelegramMessageId() == null).forEach(telegramBot::send); - } - - private static void waitSeconds(final long seconds) throws InterruptedException { - final Object lock = new Object(); - synchronized (lock) { - lock.wait(seconds * 1000); - } - } - -} diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java index fed1edd..497b5d6 100644 --- a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java @@ -1,128 +1,75 @@ package de.ph87.kleinanzeigen.kleinanzeigen; -import de.ph87.kleinanzeigen.kleinanzeigen.offer.Offer; -import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferParseException; -import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository; +import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferCreate; +import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferDto; +import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferService; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URI; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZonedDateTime; -import java.util.TimeZone; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.concurrent.TimeUnit; @Slf4j +@Service +@EnableScheduling +@RequiredArgsConstructor public class KleinanzeigenApi { private static final String VERSCHENKEN_EPPELBORN_30KM = "https://www.kleinanzeigen.de/s-zu-verschenken/66571/seite:%d/c192l339r30"; - private final OfferRepository offerRepository; + private static final int FETCH_UNTIL_DUPLICATE_MAX_PAGES = 1; - public KleinanzeigenApi(final OfferRepository offerRepository) { - this.offerRepository = offerRepository; + private final OfferService offerService; + + private final ApplicationEventPublisher publisher; + + @Scheduled(initialDelay = 0, fixedRate = 1, timeUnit = TimeUnit.MINUTES) + public void fetch() { + fetchPagesUntilDuplicate(); } - public void fetchUntilDuplicate(final int maxPageCount) { + private void fetchPagesUntilDuplicate() { int page = 0; final FetchResult totalFetchResult = new FetchResult(); - while (totalFetchResult.getUpdated() <= 0 && page <= maxPageCount) { - final FetchResult pageFetchResult = fetch(++page); + while (totalFetchResult.getUpdated() <= 0 && page < FETCH_UNTIL_DUPLICATE_MAX_PAGES) { + final FetchResult pageFetchResult = fetchPage(++page); totalFetchResult.merge(pageFetchResult); } log.debug("FetchResult: {}", totalFetchResult); } - private FetchResult fetch(final int page) { + private FetchResult fetchPage(final int page) { final FetchResult fetchResult = new FetchResult(); + final Document document; + final URI uri = URI.create(VERSCHENKEN_EPPELBORN_30KM.formatted(page)); try { - final URI uri = getPageURI(page); log.debug("Fetching page: {}", uri); - final Document document = Jsoup.parse(uri.toURL(), 3000); - for (Element article : document.select("li.ad-listitem:not(.is-topad) article.aditem")) { - final Offer offer; - try { - offer = parse(article, uri); - } catch (OfferParseException e) { - log.error("Failed to parse Offer:", e); - fetchResult.add(MergeResult.ERROR); - continue; - } - final MergeResult mergeResult = offerRepository.save(offer); - fetchResult.add(mergeResult); - } - offerRepository.flush(); + document = Jsoup.parse(uri.toURL(), 3000); } catch (IOException e) { log.error("Failed to fetch Kleinanzeigen: {}", e.toString()); + return fetchResult; } + document.select("li.ad-listitem:not(.is-topad) article.aditem").forEach(article -> tryParse(article, uri, fetchResult)); return fetchResult; } - private URI getPageURI(final int page) throws MalformedURLException { - return URI.create(VERSCHENKEN_EPPELBORN_30KM.formatted(page)); - } - - private Offer parse(final Element article, final URI uri) throws OfferParseException { + private void tryParse(final Element article, final URI uri, final FetchResult fetchResult) { try { - final String id = article.attr("data-adid"); - final String title = article.select(".text-module-begin").text(); - final String description = article.select(".aditem-main--middle--description").text(); - final ZonedDateTime date = parseDate(article.select(".aditem-main--top--right").text()); - final String articleURL = uri.resolve(article.select(".aditem-image a").attr("href")).toString(); - final String zipcode; - final String location; - final Integer distance; - final String locationString = article.select(".aditem-main--top--left").text(); - final Matcher locationMatcher = Pattern.compile("^(?\\d+) (?.+) \\((:?ca.)?\\s*(?\\d+)\\s*km\\s*\\)$").matcher(locationString); - if (!locationMatcher.find()) { - zipcode = ""; - location = locationString; - distance = null; - } else { - zipcode = locationMatcher.group("zipcode"); - location = locationMatcher.group("location"); - distance = Integer.parseInt(locationMatcher.group("distance")); - } - - final String imageURL = getImageURL(articleURL); - - return new Offer(id, date, title, zipcode, location, distance, description, articleURL, imageURL); + final OfferCreate create = new OfferCreate(article, uri); + final OfferDto dto = offerService.updateOrCreate(create, fetchResult); + publisher.publishEvent(dto); } catch (NumberFormatException e) { - throw new OfferParseException(article, e); + log.error("Failed to parse Offer:\n{}\n", article.outerHtml(), e); + fetchResult.add(MergeResult.ERROR); } } - private String getImageURL(final String articleURL) { - try { - final Document document = Jsoup.parse(URI.create(articleURL).toURL(), 3000); - final Element image = document.select(".galleryimage-element img").first(); - if (image != null) { - return image.attr("src"); - } - } catch (IOException e) { - log.error("Failed to load Article page: {}", articleURL); - } - return ""; - } - - private ZonedDateTime parseDate(final String text) { - final Matcher dayNameMatcher = Pattern.compile("(?Gestern|Heute), (?\\d+):(?\\d+)").matcher(text); - if (dayNameMatcher.find()) { - final long minusDays = dayNameMatcher.group("day").equals("Gestern") ? 1 : 0; - return ZonedDateTime.now().minusDays(minusDays).withHour(Integer.parseInt(dayNameMatcher.group("hour"))).withMinute(Integer.parseInt(dayNameMatcher.group("minute"))).withSecond(0).withNano(0); - } - - final Matcher localDateMatcher = Pattern.compile("(?\\d+).(?\\d+).(?\\d+)").matcher(text); - if (localDateMatcher.find()) { - return ZonedDateTime.of(LocalDate.of(Integer.parseInt(localDateMatcher.group("day")), Integer.parseInt(localDateMatcher.group("month")), Integer.parseInt(localDateMatcher.group("year"))), LocalTime.MIDNIGHT, TimeZone.getDefault().toZoneId()); - } - throw new NumberFormatException("Failed to parse date: " + text); - } - } diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java index dae6b71..495fc49 100644 --- a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java @@ -1,131 +1,110 @@ package de.ph87.kleinanzeigen.kleinanzeigen.offer; -import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import jakarta.annotation.Nullable; +import jakarta.persistence.*; +import lombok.*; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +@Entity @Getter +@ToString(onlyExplicitlyIncluded = true) @NoArgsConstructor public class Offer { - private String id; + @Id + @GeneratedValue + private long id; - @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + @Version + private long version; + + @NonNull + @Column(nullable = false) private ZonedDateTime first = ZonedDateTime.now(); - @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + @NonNull + @Column(nullable = false) private ZonedDateTime last = first; - @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") - private ZonedDateTime date; - - private String title; - - private String zipcode; - - private String location; - - private Integer distance; - - private String description; - - private String href; - - private String image; + @NonNull + @Column(nullable = false) + private ZonedDateTime expiry = first.plusDays(3); @Setter - private Integer telegramMessageId = null; - - private boolean ignore = false; + @Column + private boolean hide = false; + @Column private boolean remember = false; + @NonNull + @Column(nullable = false) + private String articleId; + + @NonNull + @Column(nullable = false) + private ZonedDateTime articleDate; + + @NonNull + @Column(nullable = false) + private String title; + + @NonNull + @Column(nullable = false) + private String location; + + @Column + @Nullable + private String zipcode = null; + + @Column + @Nullable + private Integer distance = null; + + @NonNull + @Column(nullable = false) + private String description; + + @NonNull + @Column(nullable = false) + private String articleURL; + + @Column + @Nullable + private String imageURL = null; + @Setter - @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") - private ZonedDateTime rememberUntil = null; + @Column + @Nullable + private Integer telegramMessageId = null; - @JsonIgnore - private boolean _deleted_ = false; - - public Offer(final String id, final ZonedDateTime date, final String title, final String zipcode, final String location, final Integer distance, final String description, final String href, final String image) { - this.id = id; - this.date = date; - this.title = title; - this.zipcode = zipcode; - this.location = location; - this.distance = distance; - this.description = description; - this.href = href; - this.image = image; + public Offer(final @NonNull OfferCreate create) { + this.articleId = create.getArticleId(); + this.articleDate = create.getArticleDate(); + update(create); } - @Override - public String toString() { - return "Offer(%s%s, %s, %s)".formatted(_deleted_ ? "[DELETED], " : "", title, calculateLocationString(), href); + public void update(final OfferCreate create) { + this.title = create.getTitle(); + this.zipcode = create.getZipcode(); + this.location = create.getLocation(); + this.distance = create.getDistance(); + this.description = create.getDescription(); + this.articleURL = create.getArticleURL(); + this.imageURL = create.getImageURL(); } - public void verifyNotDeleted() { - if (_deleted_) { - throw new RuntimeException(); + public void setRemember(final boolean newRemember) { + if (remember && !newRemember) { + final ZonedDateTime oneHour = ZonedDateTime.now().plusHours(1); + if (oneHour.isAfter(expiry)) { + expiry = oneHour; + } } - } - - public void merge(final Offer other) { - verifyNotDeleted(); - other.verifyNotDeleted(); - if (!id.equals(other.id)) { - throw new RuntimeException(); - } - this.date = other.date; - this.title = other.title; - this.zipcode = other.zipcode; - this.location = other.location; - this.distance = other.distance; - this.description = other.description; - this.href = other.href; - this.image = other.image; - this.first = other.first.isBefore(this.first) ? other.first : this.first; - this.last = other.last.isAfter(this.last) ? other.last : this.last; - other.markDeleted(); - } - - public void markDeleted() { - _deleted_ = true; - } - - public boolean _deleted_() { - return _deleted_; - } - - public String calculateLocationString() { - String result = zipcode; - if (!result.isEmpty()) { - result += " "; - } - result += location; - if (distance != null) { - result += " (" + distance + " km)"; - } - return result; - } - - public void ignore() { - ignore = true; - remember = false; - rememberUntil = null; - } - - public void setRemember(final boolean remember) { - if (remember) { - this.rememberUntil = null; - } else if (this.remember) { - this.rememberUntil = ZonedDateTime.now().plusHours(1); - } - this.remember = remember; + remember = newRemember; } } diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferCreate.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferCreate.java new file mode 100644 index 0000000..9bcbc11 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferCreate.java @@ -0,0 +1,103 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.offer; + +import jakarta.annotation.Nullable; +import lombok.Getter; +import lombok.NonNull; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import java.io.IOException; +import java.net.URI; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +@Getter +@ToString(onlyExplicitlyIncluded = true) +public class OfferCreate { + + @NonNull + @ToString.Include + private final String articleId; + + @NonNull + private final ZonedDateTime articleDate; + + @NonNull + @ToString.Include + private final String title; + + @NonNull + @ToString.Include + private final String location; + + @Nullable + private final String zipcode; + + @Nullable + private final Integer distance; + + @NonNull + private final String description; + + @NonNull + private final String articleURL; + + @Nullable + private final String imageURL; + + public OfferCreate(final Element article, final URI uri) { + articleId = article.attr("data-adid"); + title = article.select(".text-module-begin").text(); + description = article.select(".aditem-main--middle--description").text(); + articleDate = parseDate(article.select(".aditem-main--top--right").text()); + articleURL = uri.resolve(article.select(".aditem-image a").attr("href")).toString(); + final String locationString = article.select(".aditem-main--top--left").text(); + final Matcher locationMatcher = Pattern.compile("^(?\\d+) (?.+) \\((:?ca.)?\\s*(?\\d+)\\s*km\\s*\\)$").matcher(locationString); + if (!locationMatcher.find()) { + zipcode = ""; + location = locationString; + distance = null; + } else { + zipcode = locationMatcher.group("zipcode"); + location = locationMatcher.group("location"); + distance = Integer.parseInt(locationMatcher.group("distance")); + } + imageURL = getImageURL(articleURL); + } + + private ZonedDateTime parseDate(final String text) { + final Matcher dayNameMatcher = Pattern.compile("(?Gestern|Heute), (?\\d+):(?\\d+)").matcher(text); + if (dayNameMatcher.find()) { + final long minusDays = dayNameMatcher.group("day").equals("Gestern") ? 1 : 0; + return ZonedDateTime.now().minusDays(minusDays).withHour(Integer.parseInt(dayNameMatcher.group("hour"))).withMinute(Integer.parseInt(dayNameMatcher.group("minute"))).withSecond(0).withNano(0); + } + + final Matcher localDateMatcher = Pattern.compile("(?\\d+).(?\\d+).(?\\d+)").matcher(text); + if (localDateMatcher.find()) { + return ZonedDateTime.of(LocalDate.of(Integer.parseInt(localDateMatcher.group("day")), Integer.parseInt(localDateMatcher.group("month")), Integer.parseInt(localDateMatcher.group("year"))), LocalTime.MIDNIGHT, TimeZone.getDefault().toZoneId()); + } + throw new NumberFormatException("Failed to parse date: " + text); + } + + private String getImageURL(final String articleURL) { + try { + final Document document = Jsoup.parse(URI.create(articleURL).toURL(), 3000); + final Element image = document.select(".galleryimage-element img").first(); + if (image != null) { + return image.attr("src"); + } + } catch (IOException e) { + log.error("Failed to load Article page: {}", articleURL); + } + return null; + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferDto.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferDto.java new file mode 100644 index 0000000..19da634 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferDto.java @@ -0,0 +1,100 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.offer; + +import jakarta.annotation.Nullable; +import lombok.Getter; +import lombok.NonNull; +import lombok.ToString; + +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; + +@Getter +@ToString(onlyExplicitlyIncluded = true) +public class OfferDto { + + private final long id; + + private final long version; + + @NonNull + private final ZonedDateTime first; + + @NonNull + private final ZonedDateTime last; + + @NonNull + private final ZonedDateTime expiry; + + private final boolean hide; + + private final boolean remember; + + @NonNull + private final String articleId; + + @NonNull + private final ZonedDateTime articleDate; + + @NonNull + private final String title; + + @NonNull + private final String location; + + @Nullable + private final String zipcode; + + @Nullable + private final Integer distance; + + @NonNull + private final String description; + + @NonNull + private final String articleURL; + + @Nullable + private final String imageURL; + + @Nullable + private final Integer telegramMessageId; + + private final boolean _existing_; + + public OfferDto(final @NonNull Offer offer, final boolean existing) { + id = offer.getId(); + version = offer.getVersion(); + first = offer.getFirst(); + last = offer.getLast(); + expiry = offer.getExpiry(); + hide = offer.isHide(); + remember = offer.isRemember(); + + articleId = offer.getArticleId(); + articleDate = offer.getArticleDate(); + title = offer.getTitle(); + zipcode = offer.getZipcode(); + location = offer.getLocation(); + distance = offer.getDistance(); + description = offer.getDescription(); + articleURL = offer.getArticleURL(); + imageURL = offer.getImageURL(); + telegramMessageId = offer.getTelegramMessageId(); + + _existing_ = existing; + } + + public String combineLocation() { + final List list = new ArrayList<>(); + if (zipcode != null) { + list.add(zipcode); + } + list.add(location); + if (distance != null) { + list.add("(" + distance + " km)"); + } + return String.join(" ", list); + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferRepository.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferRepository.java index b19cbb0..3fd5051 100644 --- a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferRepository.java +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferRepository.java @@ -1,122 +1,17 @@ package de.ph87.kleinanzeigen.kleinanzeigen.offer; -import de.ph87.kleinanzeigen.kleinanzeigen.MergeResult; -import lombok.extern.slf4j.Slf4j; -import org.telegram.telegrambots.meta.api.objects.MaybeInaccessibleMessage; +import org.springframework.data.repository.ListCrudRepository; -import java.io.File; -import java.io.IOException; import java.time.ZonedDateTime; -import java.util.*; -import java.util.function.Consumer; +import java.util.List; +import java.util.Optional; -import static de.ph87.kleinanzeigen.JSON.objectMapper; +public interface OfferRepository extends ListCrudRepository { -@Slf4j -public class OfferRepository { + List findAllByExpiryBefore(final ZonedDateTime deadline); - private static final int KEEP_LAST_OFFERS_COUNT = 200; + Optional findByArticleId(String articleId); - private static final File FILE = new File("./offers.json"); - - private final List offers; - - private final Consumer remove; - - public OfferRepository(final Consumer remove) { - this.remove = remove; - offers = load(); - } - - private List load() { - try { - final List offers = objectMapper.readerForListOf(Offer.class).readValue(FILE); - log.info("Loaded {} offers from file: {}", offers.size(), FILE); - return offers; - } catch (IOException e) { - log.warn("Failed to load Offers from file={}: {}", FILE, e.toString()); - return new ArrayList<>(); - } - } - - public void flush() { - try { - final List removed; - synchronized (offers) { - removed = _cleanUp(); - objectMapper.writerWithDefaultPrettyPrinter().writeValue(FILE, offers); - log.debug("Wrote {} offers to file: {}", offers.size(), FILE); - } - removed.forEach(remove); - } catch (IOException e) { - log.warn("Failed to write Offers to file={}: {}", FILE, e.toString()); - } - } - - private List _cleanUp() { - if (offers.stream().anyMatch(Offer::_deleted_)) { - throw new RuntimeException(); - } - - offers.sort(Comparator.comparing(Offer::getDate)); - - final ZonedDateTime now = ZonedDateTime.now(); - final List deleted = new ArrayList<>(); - final List removable = new ArrayList<>(offers.stream().filter(offer -> !offer.isRemember() && (offer.getRememberUntil() == null || now.isAfter(offer.getRememberUntil()))).toList()); - while (!removable.isEmpty() && removable.size() > OfferRepository.KEEP_LAST_OFFERS_COUNT) { - final Offer offer = removable.removeFirst(); - offers.remove(offer); - offer.markDeleted(); - deleted.add(offer); - } - - return deleted; - } - - public MergeResult save(final Offer offer) { - synchronized (offer) { - final Optional existingOptional = offers.stream().filter(existing -> existing.getId().equals(offer.getId())).findFirst(); - if (existingOptional.isPresent()) { - existingOptional.get().merge(offer); - return MergeResult.UPDATED; - } else { - log.info("Created: {}", offer); - offers.add(offer); - return MergeResult.CREATED; - } - } - } - - public List findAll() { - synchronized (offers) { - return new ArrayList<>(offers); - } - } - - public void ignore(final MaybeInaccessibleMessage message) { - synchronized (offers) { - findByTelegramMessageId(message).ifPresent(offer -> { - offer.ignore(); - flush(); - }); - } - } - - public Optional remember(final MaybeInaccessibleMessage message, final boolean remember) { - synchronized (offers) { - final Optional optional = findByTelegramMessageId(message); - optional.ifPresent(offer -> { - offer.setRemember(remember); - flush(); - }); - return optional; - } - } - - public Optional findByTelegramMessageId(final MaybeInaccessibleMessage message) { - synchronized (offers) { - return offers.stream().filter(offer -> Objects.equals(offer.getTelegramMessageId(), message.getMessageId())).findFirst(); - } - } + Optional findByTelegramMessageId(int telegramMessageId); } diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferService.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferService.java new file mode 100644 index 0000000..d580ee9 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferService.java @@ -0,0 +1,73 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.offer; + +import de.ph87.kleinanzeigen.kleinanzeigen.FetchResult; +import de.ph87.kleinanzeigen.kleinanzeigen.MergeResult; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Optional; + +@Slf4j +@Service +@Transactional +@RequiredArgsConstructor +public class OfferService { + + private final OfferRepository repository; + + public List cleanUp() { + final List list = repository.findAllByExpiryBefore(ZonedDateTime.now()); + repository.deleteAll(list); + return list.stream().map(offer -> toDto(offer, false)).toList(); + } + + private OfferDto toDto(final Offer offer, final boolean existing) { + return new OfferDto(offer, existing); + } + + public OfferDto updateOrCreate(final OfferCreate create, final FetchResult fetchResult) { + return toDto(_updateOrCreate(create, fetchResult), true); + } + + private Offer _updateOrCreate(final OfferCreate create, final FetchResult fetchResult) { + return repository + .findByArticleId(create.getArticleId()) + .stream().peek( + existing -> { + existing.update(create); + fetchResult.add(MergeResult.UPDATED); + }) + .findFirst() + .orElseGet(() -> { + fetchResult.add(MergeResult.CREATED); + return repository.save(new Offer(create)); + } + ); + } + + @SuppressWarnings("UnusedReturnValue") + public Optional hideByTelegramMessageId(final int messageId, final boolean hide) { + return repository.findByTelegramMessageId(messageId).stream().peek(offer -> offer.setHide(hide)).findFirst().map(offer -> toDto(offer, true)); + } + + public Optional rememberByTelegramMessageId(final int messageId, final boolean remember) { + return repository.findByTelegramMessageId(messageId).stream().peek(offer -> offer.setRemember(remember)).findFirst().map(offer -> toDto(offer, true)); + } + + public Optional findByTelegramMessageId(final int messageId) { + return repository.findByTelegramMessageId(messageId).map(offer -> toDto(offer, true)); + } + + public void setTelegramMessageId(final OfferDto dto, final Integer telegramMessageId) { + findByDto(dto).ifPresent(offer -> offer.setTelegramMessageId(telegramMessageId)); + } + + private Optional findByDto(final OfferDto dto) { + return repository.findById(dto.getId()); + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/telegram/InlineCommand.java b/src/main/java/de/ph87/kleinanzeigen/telegram/InlineCommand.java index 12bba08..ff2c09b 100644 --- a/src/main/java/de/ph87/kleinanzeigen/telegram/InlineCommand.java +++ b/src/main/java/de/ph87/kleinanzeigen/telegram/InlineCommand.java @@ -1,5 +1,5 @@ package de.ph87.kleinanzeigen.telegram; public enum InlineCommand { - IGNORE, REMEMBER, UNREMEMBER + HIDE, REMEMBER, UNREMEMBER } diff --git a/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramBot.java b/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramBot.java index 8793d3e..0287760 100644 --- a/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramBot.java +++ b/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramBot.java @@ -1,57 +1,25 @@ package de.ph87.kleinanzeigen.telegram; -import com.fasterxml.jackson.core.JsonProcessingException; -import de.ph87.kleinanzeigen.kleinanzeigen.offer.Offer; -import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository; import lombok.extern.slf4j.Slf4j; import org.telegram.telegrambots.bots.TelegramLongPollingBot; import org.telegram.telegrambots.meta.TelegramBotsApi; -import org.telegram.telegrambots.meta.api.methods.send.SendPhoto; -import org.telegram.telegrambots.meta.api.methods.updatingmessages.DeleteMessages; -import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageCaption; -import org.telegram.telegrambots.meta.api.objects.*; -import org.telegram.telegrambots.meta.api.objects.replykeyboard.InlineKeyboardMarkup; -import org.telegram.telegrambots.meta.api.objects.replykeyboard.buttons.InlineKeyboardButton; +import org.telegram.telegrambots.meta.api.objects.Update; import org.telegram.telegrambots.meta.exceptions.TelegramApiException; import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -import static de.ph87.kleinanzeigen.JSON.objectMapper; +import java.util.function.Consumer; @Slf4j public class TelegramBot extends TelegramLongPollingBot { - private static final long CHAT_ID = 101138682L; - - private static final String ICON_CHECK = "✅"; - - private static final String ICON_REMOVE = "❌"; - - private final byte[] NO_IMAGE; - private final DefaultBotSession session; - private final OfferRepository offerRepository; + private final Consumer onUpdate; - public TelegramBot(final OfferRepository offerRepository) throws IOException, TelegramApiException { - super(readToken()); - this.offerRepository = offerRepository; - - final BufferedImage img = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB); - final ByteArrayOutputStream stream = new ByteArrayOutputStream(); - ImageIO.write(img, "PNG", stream); - NO_IMAGE = stream.toByteArray(); + public TelegramBot(final String token, final Consumer onUpdate) throws IOException, TelegramApiException { + super(token); + this.onUpdate = onUpdate; log.info("Starting telegram bot..."); final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class); @@ -59,12 +27,6 @@ public class TelegramBot extends TelegramLongPollingBot { log.info("Telegram bot registered."); } - private static String readToken() throws IOException { - try (final FileInputStream stream = new FileInputStream("./telegram.token")) { - return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim(); - } - } - @Override public String getBotUsername() { return "BotKleinanzeigenBot"; @@ -72,128 +34,7 @@ public class TelegramBot extends TelegramLongPollingBot { @Override public void onUpdateReceived(final Update update) { - if (update.hasMessage() && update.getMessage().hasText()) { - log.info("#{} \"{}\": {}", update.getMessage().getChat().getId(), update.getMessage().getChat().getUserName(), update.getMessage().getText()); - } else if (update.hasCallbackQuery()) { - handleCallbackQuery(update.getCallbackQuery()); - } - } - - private void handleCallbackQuery(final CallbackQuery query) { - final MaybeInaccessibleMessage message = query.getMessage(); - if (message.getChatId() != CHAT_ID) { - return; - } - try { - final InlineDto dto = objectMapper.readValue(query.getData(), InlineDto.class); - switch (dto.getCommand()) { - case IGNORE -> ignore(message); - case REMEMBER -> remember(message); - case UNREMEMBER -> unremember(message); - default -> updateMessage(message); - } - } catch (JsonProcessingException e) { - log.error("Failed to read InlineDto.", e); - } - } - - private void ignore(final MaybeInaccessibleMessage message) { - offerRepository.ignore(message); - remove(message); - } - - private void remember(final MaybeInaccessibleMessage message) { - offerRepository.remember(message, true).ifPresentOrElse( - offer -> updateMessage(message, offer), - () -> remove(message) - ); - } - - private void unremember(final MaybeInaccessibleMessage message) { - offerRepository.remember(message, false).ifPresentOrElse( - offer -> updateMessage(message, offer), - () -> remove(message) - ); - } - - private void updateMessage(final MaybeInaccessibleMessage message) { - offerRepository.findByTelegramMessageId(message).ifPresentOrElse( - offer -> updateMessage(message, offer), - () -> remove(message) - ); - } - - private void updateMessage(final MaybeInaccessibleMessage message, final Offer offer) { - try { - final EditMessageCaption edit = new EditMessageCaption(message.getChatId() + "", message.getMessageId(), null, createText(offer), createKeyboard(offer), null, null); - edit.setParseMode("Markdown"); - execute(edit); - } catch (TelegramApiException | JsonProcessingException e) { - log.error("Failed to edit Message to #{}.", message.getChatId(), e); - } - } - - public void send(final Offer offer) { - try { - final InputFile inputFile = offer.getImage().isEmpty() ? new InputFile(new ByteArrayInputStream(NO_IMAGE), "[Kein Bild]") : new InputFile(offer.getImage()); - final SendPhoto send = new SendPhoto(CHAT_ID + "", inputFile); - send.setCaption(createText(offer)); - send.setParseMode("Markdown"); - send.setReplyMarkup(createKeyboard(offer)); - final Message message = execute(send); - - offer.setTelegramMessageId(message.getMessageId()); - } catch (TelegramApiException | JsonProcessingException e) { - log.error("Failed to send Message to #{}.", CHAT_ID, e); - } - } - - private String createText(final Offer offer) { - return "[%s](%s)\n%s\n%s".formatted( - offer.getTitle().replaceAll("\\[", "(").replaceAll("]", ")"), - offer.getHref(), - offer.calculateLocationString(), - offer.getDescription() - ); - } - - private InlineKeyboardMarkup createKeyboard(final Offer offer) throws JsonProcessingException { - final InlineKeyboardMarkup markup = new InlineKeyboardMarkup(); - final ArrayList> keyboard = new ArrayList<>(); - final ArrayList row = new ArrayList<>(); - if (offer.isRemember()) { - addButton(row, ICON_CHECK + ICON_CHECK + ICON_CHECK + " Gemerkt " + ICON_CHECK + ICON_CHECK + ICON_CHECK, InlineCommand.UNREMEMBER); - } else { - addButton(row, ICON_REMOVE + " Entfernen", InlineCommand.IGNORE); - addButton(row, ICON_CHECK + " Merken", InlineCommand.REMEMBER); - } - keyboard.add(row); - markup.setKeyboard(keyboard); - return markup; - } - - private void addButton(final ArrayList row, final String caption, final InlineCommand command) throws JsonProcessingException { - final String data = objectMapper.writeValueAsString(new InlineDto(command)); - row.add(new InlineKeyboardButton(caption, null, data, null, null, null, null, null, null)); - } - - public void remove(final List offers) { - _remove(offers.stream().map(Offer::getTelegramMessageId).filter(Objects::nonNull).toList()); - } - - private void remove(final MaybeInaccessibleMessage... messages) { - _remove(Arrays.stream(messages).map(MaybeInaccessibleMessage::getMessageId).toList()); - } - - private void _remove(final List messageIds) { - if (messageIds.isEmpty()) { - return; - } - try { - execute(new DeleteMessages(CHAT_ID + "", messageIds)); - } catch (TelegramApiException e) { - log.error("Failed to remove Message to #{}.", CHAT_ID, e); - } + onUpdate.accept(update); } public void stop() { diff --git a/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramService.java b/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramService.java new file mode 100644 index 0000000..b42a1d3 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramService.java @@ -0,0 +1,211 @@ +package de.ph87.kleinanzeigen.telegram; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferDto; +import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferService; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.telegram.telegrambots.meta.api.methods.send.SendPhoto; +import org.telegram.telegrambots.meta.api.methods.updatingmessages.DeleteMessages; +import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageCaption; +import org.telegram.telegrambots.meta.api.objects.*; +import org.telegram.telegrambots.meta.api.objects.replykeyboard.InlineKeyboardMarkup; +import org.telegram.telegrambots.meta.api.objects.replykeyboard.buttons.InlineKeyboardButton; +import org.telegram.telegrambots.meta.exceptions.TelegramApiException; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TelegramService { + + private static final long CHAT_ID = 101138682L; + + private static final String ICON_CHECK = "✅"; + + private static final String ICON_REMOVE = "❌"; + + private byte[] NO_IMAGE = null; + + private final ObjectMapper objectMapper; + + private final OfferService offerService; + + private TelegramBot bot = null; + + @PostConstruct + public void postConstruct() throws IOException { + final BufferedImage img = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB); + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + ImageIO.write(img, "PNG", stream); + NO_IMAGE = stream.toByteArray(); + + try { + bot = new TelegramBot(readToken(), this::onUpdateReceived); + } catch (TelegramApiException | IOException e) { + log.error("Failed to start TelegramBot: {}", e.toString()); + } + } + + @EventListener(OfferDto.class) + public void onOfferChanged(final OfferDto offer) { + if (offer.is_existing_()) { + if (offer.getTelegramMessageId() == null) { + send(offer); + }else{ + updateMessage(offer.getTelegramMessageId(), offer.getTelegramMessageId().getChatId()); + } + } else { + remove(List.of(offer)); + } + } + + @PreDestroy + public void stop() { + log.info("Stopping Telegram bot..."); + bot.stop(); + log.info("Telegram bot stopped"); + } + + private static String readToken() throws IOException { + try (final FileInputStream stream = new FileInputStream("./telegram.token")) { + return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim(); + } + } + + private void onUpdateReceived(final Update update) { + if (update.hasMessage() && update.getMessage().hasText()) { + log.info("#{} \"{}\": {}", update.getMessage().getChat().getId(), update.getMessage().getChat().getUserName(), update.getMessage().getText()); + } else if (update.hasCallbackQuery()) { + handleCallbackQuery(update.getCallbackQuery()); + } + } + + private void handleCallbackQuery(final CallbackQuery query) { + final MaybeInaccessibleMessage message = query.getMessage(); + if (message.getChatId() != CHAT_ID) { + return; + } + try { + final InlineDto dto = objectMapper.readValue(query.getData(), InlineDto.class); + switch (dto.getCommand()) { + case HIDE -> hide(message); + case REMEMBER -> remember(message, true); + case UNREMEMBER -> remember(message, false); + default -> updateMessage(message, message.getChatId()); + } + } catch (JsonProcessingException e) { + log.error("Failed to read InlineDto.", e); + } + } + + private void hide(final MaybeInaccessibleMessage message) { + offerService.hideByTelegramMessageId(message.getMessageId(), true); + remove(message); + } + + private void remember(final MaybeInaccessibleMessage message, final boolean remember) { + offerService.rememberByTelegramMessageId(message.getMessageId(), remember).ifPresentOrElse( + offer -> updateMessage(offer, message.getChatId(), message.getMessageId()), + () -> remove(message) + ); + } + + private void updateMessage(final MaybeInaccessibleMessage message, final Long chatId) { + offerService.findByTelegramMessageId(message.getMessageId()).ifPresentOrElse( + offer -> updateMessage(offer, chatId, message.getMessageId()), + () -> remove(message) + ); + } + + private void updateMessage(final OfferDto offer, final long chatId, final int messageId) { + try { + final EditMessageCaption edit = new EditMessageCaption(chatId + "", messageId, null, createText(offer), createKeyboard(offer), null, null); + edit.setParseMode("Markdown"); + bot.execute(edit); + } catch (TelegramApiException | JsonProcessingException e) { + log.error("Failed to edit Message to #{}.", chatId, e); + } + } + + private void send(final OfferDto offer) { + try { + final InputFile inputFile = offer.getImageURL() == null ? new InputFile(new ByteArrayInputStream(NO_IMAGE), "[Kein Bild]") : new InputFile(offer.getImageURL()); + final SendPhoto send = new SendPhoto(CHAT_ID + "", inputFile); + log.debug("{}", send); + send.setCaption(createText(offer)); + send.setParseMode("Markdown"); + send.setReplyMarkup(createKeyboard(offer)); + final Message message = bot.execute(send); + + offerService.setTelegramMessageId(offer, message.getMessageId()); + } catch (TelegramApiException | JsonProcessingException e) { + log.error("Failed to send Message to #{}.", CHAT_ID, e); + } + } + + private String createText(final OfferDto offer) { + return "[%s](%s)\n%s\n%s".formatted( + offer.getTitle().replaceAll("\\[", "(").replaceAll("]", ")"), + offer.getArticleURL(), + offer.combineLocation(), + offer.getDescription() + ); + } + + private InlineKeyboardMarkup createKeyboard(final OfferDto offer) throws JsonProcessingException { + final InlineKeyboardMarkup markup = new InlineKeyboardMarkup(); + final ArrayList> keyboard = new ArrayList<>(); + final ArrayList row = new ArrayList<>(); + if (offer.isRemember()) { + addButton(row, ICON_CHECK + ICON_CHECK + ICON_CHECK + " Gemerkt " + ICON_CHECK + ICON_CHECK + ICON_CHECK, InlineCommand.UNREMEMBER); + } else { + addButton(row, ICON_REMOVE + " Entfernen", InlineCommand.HIDE); + addButton(row, ICON_CHECK + " Merken", InlineCommand.REMEMBER); + } + keyboard.add(row); + markup.setKeyboard(keyboard); + return markup; + } + + private void addButton(final ArrayList row, final String caption, final InlineCommand command) throws JsonProcessingException { + final String data = objectMapper.writeValueAsString(new InlineDto(command)); + row.add(new InlineKeyboardButton(caption, null, data, null, null, null, null, null, null)); + } + + private void remove(final List offers) { + _remove(offers.stream().map(OfferDto::getTelegramMessageId).filter(Objects::nonNull).toList()); + } + + private void remove(final MaybeInaccessibleMessage... messages) { + _remove(Arrays.stream(messages).map(MaybeInaccessibleMessage::getMessageId).toList()); + } + + private void _remove(final List messageIds) { + if (messageIds.isEmpty()) { + return; + } + try { + bot.execute(new DeleteMessages(CHAT_ID + "", messageIds)); + } catch (TelegramApiException e) { + log.error("Failed to remove Message to #{}.", CHAT_ID, e); + } + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..f9fffb6 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,10 @@ +logging.level.root=WARN +logging.level.de.ph87=INFO +#- +spring.jpa.hibernate.naming.implicit-strategy=org.hibernate.boot.model.naming.ImplicitNamingStrategyComponentPathImpl +spring.jpa.hibernate.ddl-auto=update +spring.jpa.open-in-view=false +#- +spring.jackson.serialization.indent_output=true +#- +spring.main.banner-mode=off