From d25093dd441c1db524c0c7b38fd17ff503a4058b Mon Sep 17 00:00:00 2001 From: "hehihoho3@gmail.com" Date: Fri, 4 Sep 2026 16:12:01 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=EC=9C=A0=ED=86=B5=20=EC=86=8C=EC=85=9C?= =?UTF-8?q?=20=ED=94=BC=EB=93=9C=EC=99=80=20=ED=85=94=EB=A0=88=EA=B7=B8?= =?UTF-8?q?=EB=9E=A8=20=EC=95=8C=EB=A6=BC=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 1 + .../domain/channel/FeedAlertService.java | 2 + .../yanalyst/domain/channel/FeedTopic.java | 3 +- .../shortform/ShortformSubtitleService.java | 2 + .../domain/social/FacebookPageFeedClient.java | 151 +++++++++++++++ .../yanalyst/domain/social/SocialChannel.java | 97 ++++++++++ .../social/SocialChannelRepository.java | 12 ++ .../domain/social/SocialFeedAlertService.java | 62 ++++++ .../domain/social/SocialFeedBatch.java | 10 + .../domain/social/SocialFeedClient.java | 6 + .../domain/social/SocialFeedController.java | 52 +++++ .../domain/social/SocialFeedService.java | 179 ++++++++++++++++++ .../domain/social/SocialPlatform.java | 85 +++++++++ .../yanalyst/domain/social/SocialVideo.java | 59 ++++++ .../domain/social/SocialVideoCandidate.java | 11 ++ .../domain/social/SocialVideoRepository.java | 19 ++ .../domain/social/YtDlpSocialFeedClient.java | 131 +++++++++++++ .../com/hlab/yanalyst/web/WebController.java | 6 + .../resources/application-local.yml.example | 3 + src/main/resources/application.yml | 12 ++ .../templates/distribution-feed.html | 67 +++++++ .../resources/templates/layout/sidebar.html | 3 + .../domain/channel/FeedAlertServiceTest.java | 51 +++++ .../ShortformSubtitleServiceTest.java | 20 ++ .../social/FacebookPageFeedClientTest.java | 35 ++++ .../domain/social/SocialFeedServiceTest.java | 48 +++++ .../domain/social/SocialPlatformTest.java | 35 ++++ .../social/YtDlpSocialFeedClientTest.java | 30 +++ 28 files changed, 1191 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClient.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialChannel.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialChannelRepository.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialFeedAlertService.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialFeedBatch.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialFeedClient.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialFeedController.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialFeedService.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialPlatform.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialVideo.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialVideoCandidate.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/SocialVideoRepository.java create mode 100644 src/main/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClient.java create mode 100644 src/main/resources/templates/distribution-feed.html create mode 100644 src/test/java/com/hlab/yanalyst/domain/channel/FeedAlertServiceTest.java create mode 100644 src/test/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClientTest.java create mode 100644 src/test/java/com/hlab/yanalyst/domain/social/SocialFeedServiceTest.java create mode 100644 src/test/java/com/hlab/yanalyst/domain/social/SocialPlatformTest.java create mode 100644 src/test/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClientTest.java diff --git a/build.gradle b/build.gradle index d5efde0..352202f 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-validation' implementation 'org.springframework.boot:spring-boot-starter-thymeleaf' implementation 'nz.net.ultraq.thymeleaf:thymeleaf-layout-dialect' + implementation 'org.jsoup:jsoup:1.23.2' // Swagger (SpringDoc) — 2.7.0+ is required for Spring Boot 3.4 (2.3.0 throws NoSuchMethodError on /v3/api-docs) implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.7.0' diff --git a/src/main/java/com/hlab/yanalyst/domain/channel/FeedAlertService.java b/src/main/java/com/hlab/yanalyst/domain/channel/FeedAlertService.java index 2a53f3a..9532681 100644 --- a/src/main/java/com/hlab/yanalyst/domain/channel/FeedAlertService.java +++ b/src/main/java/com/hlab/yanalyst/domain/channel/FeedAlertService.java @@ -37,6 +37,8 @@ public class FeedAlertService { LocalDateTime now = LocalDateTime.now(); List golden = newVideos.stream() + // 정치 피드는 수집·화면 표시만 유지하고 텔레그램에서는 제외한다. + .filter(v -> v.feedTopicOrDefault() != FeedTopic.POLITICS) .filter(v -> FeedBadges.isGoldenTime(v.getPublishedAt(), now)) .sorted(Comparator.comparing(ChannelVideo::getPublishedAt).reversed()) .toList(); diff --git a/src/main/java/com/hlab/yanalyst/domain/channel/FeedTopic.java b/src/main/java/com/hlab/yanalyst/domain/channel/FeedTopic.java index 69e234d..1d941ec 100644 --- a/src/main/java/com/hlab/yanalyst/domain/channel/FeedTopic.java +++ b/src/main/java/com/hlab/yanalyst/domain/channel/FeedTopic.java @@ -2,7 +2,8 @@ package com.hlab.yanalyst.domain.channel; public enum FeedTopic { ENTERTAINMENT, - POLITICS; + POLITICS, + DISTRIBUTION; public static FeedTopic normalize(String value) { if (value == null || value.isBlank()) return ENTERTAINMENT; diff --git a/src/main/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleService.java b/src/main/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleService.java index df4e250..c837e23 100644 --- a/src/main/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleService.java +++ b/src/main/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleService.java @@ -70,6 +70,7 @@ public class ShortformSubtitleService { } private void notifyReady(ShortformJob job) { + if (job.categoryOrDefault() == ShortformCategory.POLITICS) return; telegramNotifier.sendMessage("✅ 숏폼 자막 생성 완료\n" + "분류: " + categoryLabel(job) + "\n" + "영상: " + job.getVideoId() + "\n" @@ -78,6 +79,7 @@ public class ShortformSubtitleService { } private void notifyFailure(ShortformJob job, String reason) { + if (job.categoryOrDefault() == ShortformCategory.POLITICS) return; telegramNotifier.sendMessage("❌ 숏폼 자막 생성 실패\n" + "분류: " + categoryLabel(job) + "\n" + "영상: " + job.getVideoId() + "\n" diff --git a/src/main/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClient.java b/src/main/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClient.java new file mode 100644 index 0000000..67c67cf --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClient.java @@ -0,0 +1,151 @@ +package com.hlab.yanalyst.domain.social; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Facebook Page Plugin의 공개 타임라인에서 릴 목록을 읽는다. 로그인·Graph API 토큰이 필요 없다. */ +@Component +@RequiredArgsConstructor +public class FacebookPageFeedClient implements SocialFeedClient { + private static final String PAGE_PLUGIN_APP_ID = "776730922422337"; + private static final Pattern REEL_ID = Pattern.compile("/(?:reel|videos)/(\\d+)"); + + private final RestTemplate restTemplate; + private final ObjectMapper objectMapper; + + @Value("${hlab.social-feed.max-items:25}") + private int maxItems; + + @Override + public boolean supports(SocialPlatform platform) { + return platform == SocialPlatform.FACEBOOK; + } + + @Override + public SocialFeedBatch fetch(SocialChannel channel) { + try { + Map config = new LinkedHashMap<>(); + config.put("app_id", PAGE_PLUGIN_APP_ID); + config.put("href", channel.getSourceUrl()); + config.put("width", 500); + config.put("height", 800); + config.put("has_cta", true); + config.put("has_small_header", false); + config.put("has_adapt_container_width", true); + config.put("has_cover", true); + config.put("has_posts", false); + config.put("tabs", "timeline"); + config.put("can_personalize", false); + config.put("is_xfbml", false); + config.put("referer_uri", ""); + + URI uri = UriComponentsBuilder.fromHttpUrl("https://www.facebook.com/platform/plugin/tab/renderer/") + .queryParam("key", "timeline") + .queryParam("config_json", objectMapper.writeValueAsString(config)) + .queryParam("__a", "1") + .build().encode().toUri(); + HttpHeaders headers = new HttpHeaders(); + headers.set(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"); + headers.set(HttpHeaders.ACCEPT_LANGUAGE, "ko-KR,ko;q=0.9,en;q=0.8"); + headers.set("X-Requested-With", "XMLHttpRequest"); + ResponseEntity response = restTemplate.exchange(uri, HttpMethod.GET, + new HttpEntity<>(headers), String.class); + if (!response.getStatusCode().is2xxSuccessful() || response.getBody() == null) { + throw new IllegalStateException("Facebook 공개 피드 응답이 비어 있습니다."); + } + return parseResponse(response.getBody(), Math.max(1, maxItems)); + } catch (Exception e) { + if (e instanceof IllegalStateException state) throw state; + throw new IllegalStateException("Facebook 공개 피드를 읽지 못했습니다: " + friendly(e), e); + } + } + + SocialFeedBatch parseResponse(String body, int limit) { + try { + String json = body == null ? "" : body.strip(); + if (json.startsWith("for (;;);")) json = json.substring("for (;;);".length()); + JsonNode root = objectMapper.readTree(json); + String markup = root.path("payload").path("content").path("markup").path("__html").asText(""); + if (markup.isBlank()) throw new IllegalStateException("Facebook 공개 피드에 표시할 게시물이 없습니다."); + return parseMarkup(markup, limit); + } catch (IllegalStateException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException("Facebook 공개 피드 형식이 변경되었습니다.", e); + } + } + + static SocialFeedBatch parseMarkup(String markup, int limit) { + Document document = Jsoup.parse(markup, "https://www.facebook.com"); + LinkedHashMap unique = new LinkedHashMap<>(); + String channelTitle = null; + + for (Element post : document.select("div[data-fte=1][data-ftr=1]")) { + Element reelLink = post.selectFirst("a[href*=/reel/], a[href*=/videos/]"); + if (reelLink == null) continue; + Matcher matcher = REEL_ID.matcher(reelLink.attr("href")); + if (!matcher.find()) continue; + String id = matcher.group(1); + if (unique.containsKey(id)) continue; + + Element titleElement = post.selectFirst("[data-testid=post_message]"); + String title = titleElement == null ? "(제목 없음)" : titleElement.text().strip(); + Element timeElement = post.selectFirst("abbr[data-utime]"); + LocalDateTime publishedAt = parseUnixTime(timeElement == null ? null : timeElement.attr("data-utime")); + String thumbnail = findThumbnail(post); + unique.put(id, new SocialVideoCandidate(id, "https://www.facebook.com/reel/" + id, + title.isBlank() ? "(제목 없음)" : title, thumbnail, publishedAt)); + + if (channelTitle == null) { + Element titleNode = post.selectFirst("._2_79, a[title]"); + if (titleNode != null && !titleNode.text().isBlank()) channelTitle = titleNode.text().strip(); + if ((channelTitle == null || channelTitle.isBlank()) && titleNode != null) { + channelTitle = titleNode.attr("title").strip(); + } + } + if (unique.size() >= limit) break; + } + return new SocialFeedBatch(channelTitle, null, new ArrayList<>(unique.values())); + } + + private static String findThumbnail(Element post) { + for (Element image : post.select("img[src]")) { + String src = image.absUrl("src"); + String classes = image.className(); + if (!src.isBlank() && !src.contains("s50x50") && !classes.contains("_s0")) return src; + } + return null; + } + + private static LocalDateTime parseUnixTime(String value) { + if (value == null || value.isBlank()) return null; + try { + return LocalDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(value)), ZoneId.systemDefault()); + } catch (NumberFormatException ignored) { + return null; + } + } + + private static String friendly(Exception e) { + String message = e.getMessage(); + if (message == null || message.isBlank()) return e.getClass().getSimpleName(); + return message.length() <= 500 ? message : message.substring(0, 500); + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialChannel.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialChannel.java new file mode 100644 index 0000000..f3a32ca --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialChannel.java @@ -0,0 +1,97 @@ +package com.hlab.yanalyst.domain.social; + +import com.hlab.yanalyst.domain.channel.FeedTopic; +import jakarta.persistence.*; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "social_channels", uniqueConstraints = + @UniqueConstraint(name = "uk_social_channel_source_url", columnNames = "source_url")) +@Getter +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@EntityListeners(AuditingEntityListener.class) +public class SocialChannel { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Enumerated(EnumType.STRING) + @Column(nullable = false, length = 20) + private SocialPlatform platform; + + @Column(name = "source_url", nullable = false, length = 2000) + private String sourceUrl; + + @Column(name = "external_channel_id", length = 200) + private String externalChannelId; + + @Column(nullable = false, length = 300) + private String title; + + @Enumerated(EnumType.STRING) + @Column(nullable = false, length = 20) + private FeedTopic topic; + + @Column(name = "notification_enabled", nullable = false) + private boolean notificationEnabled; + + @Column(name = "last_collected_at") + private LocalDateTime lastCollectedAt; + + @Column(name = "last_found", nullable = false) + private int lastFound; + + @Column(name = "fail_count", nullable = false) + private int failCount; + + @Column(name = "last_error", columnDefinition = "TEXT") + private String lastError; + + @CreatedDate + @Column(updatable = false) + private LocalDateTime createdAt; + + @LastModifiedDate + private LocalDateTime updatedAt; + + public static SocialChannel register(String url, String topic, boolean notificationEnabled) { + String normalizedUrl = SocialPlatform.normalizeChannelUrl(url); + SocialChannel channel = new SocialChannel(); + channel.platform = SocialPlatform.fromUrl(normalizedUrl); + channel.sourceUrl = normalizedUrl; + channel.title = SocialPlatform.displayNameFromUrl(normalizedUrl); + channel.topic = FeedTopic.normalize(topic); + channel.notificationEnabled = notificationEnabled; + return channel; + } + + public void updateMetadata(String title, String externalChannelId) { + if (title != null && !title.isBlank()) this.title = title.strip(); + if (externalChannelId != null && !externalChannelId.isBlank()) this.externalChannelId = externalChannelId.strip(); + } + + public void configure(String topic, boolean notificationEnabled) { + this.topic = FeedTopic.normalize(topic); + this.notificationEnabled = notificationEnabled; + } + + public void collectionSucceeded(int found) { + this.lastCollectedAt = LocalDateTime.now(); + this.lastFound = Math.max(0, found); + this.failCount = 0; + this.lastError = null; + } + + public void collectionFailed(String message) { + this.failCount++; + this.lastError = message == null || message.isBlank() ? "수집에 실패했습니다." : message; + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialChannelRepository.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialChannelRepository.java new file mode 100644 index 0000000..82e482c --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialChannelRepository.java @@ -0,0 +1,12 @@ +package com.hlab.yanalyst.domain.social; + +import com.hlab.yanalyst.domain.channel.FeedTopic; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface SocialChannelRepository extends JpaRepository { + Optional findBySourceUrl(String sourceUrl); + List findAllByTopicOrderByIdAsc(FeedTopic topic); +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedAlertService.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedAlertService.java new file mode 100644 index 0000000..8df2cec --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedAlertService.java @@ -0,0 +1,62 @@ +package com.hlab.yanalyst.domain.social; + +import com.hlab.yanalyst.domain.channel.FeedTopic; +import com.hlab.yanalyst.global.notify.TelegramNotifier; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Comparator; +import java.util.List; + +@Slf4j +@Service +@RequiredArgsConstructor +public class SocialFeedAlertService { + private static final int MAX_LINES = 8; + private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("M/d HH:mm"); + + private final TelegramNotifier telegramNotifier; + + public int notifyNewVideos(SocialChannel channel, List newVideos) { + if (channel == null || channel.getTopic() == FeedTopic.POLITICS || !channel.isNotificationEnabled() + || newVideos == null || newVideos.isEmpty() || !telegramNotifier.isEnabled()) return 0; + + List sorted = newVideos.stream() + .sorted(Comparator.comparing(SocialVideo::getPublishedAt, + Comparator.nullsLast(Comparator.reverseOrder()))) + .toList(); + String topicEmoji = channel.getTopic() == FeedTopic.DISTRIBUTION ? "📦" : "🎬"; + String topicName = channel.getTopic() == FeedTopic.DISTRIBUTION ? "유통" : "예능"; + StringBuilder message = new StringBuilder(topicEmoji).append(" ").append(topicName) + .append(" 새 영상 · ").append(esc(channel.getTitle())).append(" ") + .append(sorted.size()).append("건"); + + int shown = Math.min(sorted.size(), MAX_LINES); + for (int i = 0; i < shown; i++) { + SocialVideo video = sorted.get(i); + message.append("\n\n• ") + .append(esc(clip(video.getTitle()))).append(""); + if (video.getPublishedAt() != null) message.append(" · ").append(TIME.format(video.getPublishedAt())); + } + if (sorted.size() > shown) message.append("\n\n…외 ").append(sorted.size() - shown).append("건"); + message.append("\n\nhttps://h-lab.tolag.shop/distribution-feed"); + + boolean sent = telegramNotifier.sendMessage(message.toString()); + log.info("[SocialFeed] 텔레그램 알림: channel={}, sent={}, count={}", channel.getId(), sent, sorted.size()); + return sent ? sorted.size() : 0; + } + + private static String clip(String text) { + if (text == null || text.isBlank()) return "(제목 없음)"; + return text.length() <= 90 ? text : text.substring(0, 89) + "…"; + } + + private static String esc(String text) { + if (text == null) return "-"; + return text.replace("&", "&").replace("<", "<").replace(">", ">") + .replace("\"", """); + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedBatch.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedBatch.java new file mode 100644 index 0000000..64c65de --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedBatch.java @@ -0,0 +1,10 @@ +package com.hlab.yanalyst.domain.social; + +import java.util.List; + +public record SocialFeedBatch(String channelTitle, String externalChannelId, + List videos) { + public SocialFeedBatch { + videos = videos == null ? List.of() : List.copyOf(videos); + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedClient.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedClient.java new file mode 100644 index 0000000..c89f22a --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedClient.java @@ -0,0 +1,6 @@ +package com.hlab.yanalyst.domain.social; + +public interface SocialFeedClient { + boolean supports(SocialPlatform platform); + SocialFeedBatch fetch(SocialChannel channel); +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedController.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedController.java new file mode 100644 index 0000000..22acac9 --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedController.java @@ -0,0 +1,52 @@ +package com.hlab.yanalyst.domain.social; + +import com.hlab.yanalyst.domain.channel.FeedTopic; +import com.hlab.yanalyst.global.common.ApiResponse; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.*; + +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/api/social-feed") +@RequiredArgsConstructor +public class SocialFeedController { + private final SocialFeedService service; + + public record AddChannelRequest(@NotBlank(message = "채널 URL을 입력하세요.") String url, + String topic, Boolean notificationEnabled) { } + + @GetMapping("/channels") + public ApiResponse> channels( + @RequestParam(defaultValue = "DISTRIBUTION") String topic) { + return ApiResponse.ok(service.channels(topic)); + } + + @PostMapping("/channels") + public ApiResponse addChannel(@Valid @RequestBody AddChannelRequest request) { + String topic = request.topic() == null ? FeedTopic.DISTRIBUTION.name() : request.topic(); + return ApiResponse.created(service.addChannel(request.url(), topic, + request.notificationEnabled() == null || request.notificationEnabled())); + } + + @DeleteMapping("/channels/{id}") + public ApiResponse removeChannel(@PathVariable Long id) { + service.removeChannel(id); + return ApiResponse.ok(null); + } + + @GetMapping("/videos") + public ApiResponse> videos( + @RequestParam(defaultValue = "DISTRIBUTION") String topic) { + return ApiResponse.ok(service.videos(topic)); + } + + @PostMapping("/collect") + public ApiResponse> collect( + @RequestParam(defaultValue = "DISTRIBUTION") String topic) { + return ApiResponse.ok(service.collect(topic)); + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedService.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedService.java new file mode 100644 index 0000000..8c673a6 --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialFeedService.java @@ -0,0 +1,179 @@ +package com.hlab.yanalyst.domain.social; + +import com.hlab.yanalyst.domain.channel.FeedTopic; +import jakarta.persistence.EntityNotFoundException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.data.domain.PageRequest; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.*; + +@Slf4j +@Service +@RequiredArgsConstructor +public class SocialFeedService { + private static final int MAX_FEED_ITEMS = 200; + + private final SocialChannelRepository channelRepository; + private final SocialVideoRepository videoRepository; + private final List clients; + private final SocialFeedAlertService alertService; + + @Value("${hlab.social-feed.enabled:true}") private boolean enabled; + @Value("${hlab.social-feed.seed.phutunet-enabled:true}") private boolean phutunetSeedEnabled; + @Value("${hlab.social-feed.seed.phutunet-url:https://www.facebook.com/phutunet/reels/}") private String phutunetUrl; + + public record ChannelDto(Long id, String platform, String sourceUrl, String externalChannelId, + String title, String topic, boolean notificationEnabled, + LocalDateTime lastCollectedAt, int lastFound, int failCount, String lastError) { + static ChannelDto from(SocialChannel channel) { + return new ChannelDto(channel.getId(), channel.getPlatform().name(), channel.getSourceUrl(), + channel.getExternalChannelId(), channel.getTitle(), channel.getTopic().name(), + channel.isNotificationEnabled(), channel.getLastCollectedAt(), channel.getLastFound(), + channel.getFailCount(), channel.getLastError()); + } + } + + public record VideoDto(Long id, String externalVideoId, String url, String title, String thumbnailUrl, + LocalDateTime publishedAt, LocalDateTime collectedAt, + String channelTitle, String platform, String topic) { + static VideoDto from(SocialVideo video) { + SocialChannel channel = video.getChannel(); + return new VideoDto(video.getId(), video.getExternalVideoId(), video.getUrl(), video.getTitle(), + video.getThumbnailUrl(), video.getPublishedAt(), video.getCollectedAt(), channel.getTitle(), + channel.getPlatform().name(), channel.getTopic().name()); + } + } + + @EventListener(ApplicationReadyEvent.class) + @Transactional + public void registerDefaultChannels() { + if (!phutunetSeedEnabled || phutunetUrl == null || phutunetUrl.isBlank()) return; + String normalized = SocialPlatform.normalizeChannelUrl(phutunetUrl); + if (channelRepository.findBySourceUrl(normalized).isPresent()) return; + SocialChannel channel = SocialChannel.register(normalized, FeedTopic.DISTRIBUTION.name(), true); + channel.updateMetadata("Phutu Crafts", "321133534899178"); + channelRepository.save(channel); + log.info("[SocialFeed] 기본 Facebook 유통 채널 등록: {}", normalized); + } + + @Scheduled(cron = "${hlab.social-feed.cron:0 */15 * * * *}") + public void scheduledCollect() { + if (!enabled) return; + log.info("[SocialFeed] 자동 수집 완료: {}", collect(FeedTopic.DISTRIBUTION.name())); + } + + @Transactional(readOnly = true) + public List channels(String topic) { + return channelRepository.findAllByTopicOrderByIdAsc(FeedTopic.normalize(topic)).stream() + .map(ChannelDto::from).toList(); + } + + @Transactional(readOnly = true) + public List videos(String topic) { + return videoRepository.findFeed(FeedTopic.normalize(topic), PageRequest.of(0, MAX_FEED_ITEMS)).stream() + .map(VideoDto::from).toList(); + } + + @Transactional + public ChannelDto addChannel(String url, String topic, boolean notificationEnabled) { + String normalized = SocialPlatform.normalizeChannelUrl(url); + SocialChannel channel = channelRepository.findBySourceUrl(normalized) + .orElseGet(() -> SocialChannel.register(normalized, topic, notificationEnabled)); + channel.configure(topic, notificationEnabled); + return ChannelDto.from(channelRepository.save(channel)); + } + + @Transactional + public void removeChannel(Long id) { + SocialChannel channel = getChannel(id); + videoRepository.deleteAllByChannelId(id); + channelRepository.delete(channel); + } + + public Map collect(String topic) { + FeedTopic normalized = FeedTopic.normalize(topic); + List channels = channelRepository.findAllByTopicOrderByIdAsc(normalized); + int succeeded = 0, failed = 0, saved = 0, alerted = 0; + for (SocialChannel channel : channels) { + try { + CollectionResult result = collectOne(channel.getId()); + succeeded++; + saved += result.saved(); + alerted += result.alerted(); + } catch (Exception e) { + failed++; + markFailure(channel.getId(), friendly(e)); + log.warn("[SocialFeed] 채널 수집 실패: id={}, url={}", channel.getId(), channel.getSourceUrl(), e); + } + } + Map summary = new LinkedHashMap<>(); + summary.put("channels", channels.size()); + summary.put("collected", succeeded); + summary.put("savedVideos", saved); + summary.put("alerted", alerted); + summary.put("failed", failed); + return summary; + } + + @Transactional + public CollectionResult collectOne(Long channelId) { + SocialChannel channel = getChannel(channelId); + LocalDateTime previousCollection = channel.getLastCollectedAt(); + boolean baselineCollection = previousCollection == null; + SocialFeedClient client = clients.stream().filter(c -> c.supports(channel.getPlatform())).findFirst() + .orElseThrow(() -> new IllegalStateException(channel.getPlatform() + " 수집기가 없습니다.")); + SocialFeedBatch batch = client.fetch(channel); + channel.updateMetadata(batch.channelTitle(), batch.externalChannelId()); + + List inserted = new ArrayList<>(); + for (SocialVideoCandidate candidate : batch.videos()) { + if (candidate.externalVideoId() == null || candidate.externalVideoId().isBlank() + || candidate.url() == null || candidate.url().isBlank() + || videoRepository.existsByChannelIdAndExternalVideoId(channelId, candidate.externalVideoId())) { + continue; + } + inserted.add(videoRepository.save(SocialVideo.from(channel, candidate))); + } + channel.collectionSucceeded(inserted.size()); + channelRepository.save(channel); + // 최초 수집은 현재 목록을 기준점으로만 저장한다. 다음 수집부터 생긴 항목만 알린다. + List alertCandidates = baselineCollection ? List.of() : inserted.stream() + // 플러그인 목록 순서가 바뀌며 뒤늦게 발견된 과거 영상을 신규로 오인하지 않는다. + .filter(video -> video.getPublishedAt() == null + || !video.getPublishedAt().isBefore(previousCollection.minusMinutes(5))) + .toList(); + int alerted = baselineCollection ? 0 : alertService.notifyNewVideos(channel, alertCandidates); + return new CollectionResult(inserted.size(), alerted, baselineCollection); + } + + public record CollectionResult(int saved, int alerted, boolean baselineCollection) { } + + @Transactional + protected void markFailure(Long channelId, String message) { + channelRepository.findById(channelId).ifPresent(channel -> { + channel.collectionFailed(message); + channelRepository.save(channel); + }); + } + + private SocialChannel getChannel(Long id) { + return channelRepository.findById(id) + .orElseThrow(() -> new EntityNotFoundException("소셜 채널을 찾을 수 없습니다: " + id)); + } + + private static String friendly(Exception e) { + Throwable cause = e; + while (cause.getCause() != null && cause.getMessage() == null) cause = cause.getCause(); + String message = cause.getMessage(); + if (message == null || message.isBlank()) return "수집에 실패했습니다."; + return message.length() <= 1200 ? message : message.substring(message.length() - 1200); + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialPlatform.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialPlatform.java new file mode 100644 index 0000000..2d10fbb --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialPlatform.java @@ -0,0 +1,85 @@ +package com.hlab.yanalyst.domain.social; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Locale; +import java.util.Set; + +/** 유통 소재 채널에서 지원하는 소셜 플랫폼과 URL 검증 규칙. */ +public enum SocialPlatform { + FACEBOOK(Set.of("facebook.com", "fb.watch")), + INSTAGRAM(Set.of("instagram.com")), + TIKTOK(Set.of("tiktok.com")), + YOUTUBE(Set.of("youtube.com", "youtu.be")); + + private final Set hosts; + + SocialPlatform(Set hosts) { + this.hosts = hosts; + } + + public static SocialPlatform fromUrl(String value) { + URI uri = parse(value); + String host = uri.getHost().toLowerCase(Locale.ROOT); + for (SocialPlatform platform : values()) { + if (platform.hosts.stream().anyMatch(allowed -> host.equals(allowed) || host.endsWith("." + allowed))) { + return platform; + } + } + throw new IllegalArgumentException("Facebook, Instagram, TikTok, YouTube 채널 URL만 등록할 수 있습니다."); + } + + public static String normalizeChannelUrl(String value) { + URI uri = parse(value); + SocialPlatform platform = fromUrl(value); + String path = uri.getPath() == null || uri.getPath().isBlank() ? "/" : uri.getPath(); + validateChannelPath(platform, path); + while (path.length() > 1 && path.endsWith("/")) path = path.substring(0, path.length() - 1); + // Facebook Page Plugin은 /페이지/reels 주소를 직접 받으면 빈 응답을 내므로 페이지 본주소로 맞춘다. + if (platform == FACEBOOK) path = path.replaceFirst("(?i)/(?:reels|videos)$", ""); + try { + return new URI("https", null, uri.getHost().toLowerCase(Locale.ROOT), -1, path, null, null).toString(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("올바른 채널 URL을 입력하세요."); + } + } + + static String displayNameFromUrl(String normalizedUrl) { + String path = URI.create(normalizedUrl).getPath(); + String[] segments = path.split("/"); + for (String segment : segments) { + if (!segment.isBlank() && !segment.equalsIgnoreCase("reels") && !segment.equalsIgnoreCase("videos")) { + return segment.startsWith("@") ? segment.substring(1) : segment; + } + } + return fromUrl(normalizedUrl).name(); + } + + private static URI parse(String value) { + if (value == null || value.isBlank()) throw new IllegalArgumentException("채널 URL을 입력하세요."); + if (value.trim().length() > 2000) throw new IllegalArgumentException("URL은 2,000자 이하여야 합니다."); + try { + URI uri = URI.create(value.trim()); + String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); + if (!(scheme.equals("http") || scheme.equals("https")) || uri.getHost() == null || uri.getUserInfo() != null) { + throw new IllegalArgumentException("HTTP(S) 채널 URL을 입력하세요."); + } + return uri; + } catch (IllegalArgumentException e) { + if (e.getMessage() != null && (e.getMessage().contains("HTTP(S)") || e.getMessage().contains("2,000"))) throw e; + throw new IllegalArgumentException("올바른 채널 URL을 입력하세요."); + } + } + + private static void validateChannelPath(SocialPlatform platform, String path) { + String lower = path.toLowerCase(Locale.ROOT); + boolean individualContent = switch (platform) { + case FACEBOOK -> lower.matches(".*/(?:reel|watch)/.*") || lower.matches(".*/videos/\\d+.*"); + case INSTAGRAM -> lower.matches("/(?:p|reel|stories)/.*"); + case TIKTOK -> lower.contains("/video/"); + case YOUTUBE -> lower.equals("/watch") || lower.startsWith("/shorts/") || lower.startsWith("/embed/") + || lower.startsWith("/live/") || platform == YOUTUBE && path.equals("/"); + }; + if (individualContent) throw new IllegalArgumentException("개별 영상이 아닌 채널·프로필 URL을 입력하세요."); + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialVideo.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialVideo.java new file mode 100644 index 0000000..5e3cbb1 --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialVideo.java @@ -0,0 +1,59 @@ +package com.hlab.yanalyst.domain.social; + +import jakarta.persistence.*; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "social_videos", + uniqueConstraints = @UniqueConstraint(name = "uk_social_video_channel_external", + columnNames = {"channel_id", "external_video_id"}), + indexes = @Index(name = "idx_social_video_published", columnList = "published_at")) +@Getter +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@EntityListeners(AuditingEntityListener.class) +public class SocialVideo { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "channel_id", nullable = false) + private SocialChannel channel; + + @Column(name = "external_video_id", nullable = false, length = 300) + private String externalVideoId; + + @Column(nullable = false, length = 2000) + private String url; + + @Column(nullable = false, columnDefinition = "TEXT") + private String title; + + @Column(name = "thumbnail_url", length = 4000) + private String thumbnailUrl; + + @Column(name = "published_at") + private LocalDateTime publishedAt; + + @CreatedDate + @Column(name = "collected_at", updatable = false) + private LocalDateTime collectedAt; + + public static SocialVideo from(SocialChannel channel, SocialVideoCandidate candidate) { + SocialVideo video = new SocialVideo(); + video.channel = channel; + video.externalVideoId = candidate.externalVideoId(); + video.url = candidate.url(); + video.title = candidate.title() == null || candidate.title().isBlank() ? "(제목 없음)" : candidate.title(); + video.thumbnailUrl = candidate.thumbnailUrl(); + video.publishedAt = candidate.publishedAt(); + return video; + } +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialVideoCandidate.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialVideoCandidate.java new file mode 100644 index 0000000..fa243d2 --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialVideoCandidate.java @@ -0,0 +1,11 @@ +package com.hlab.yanalyst.domain.social; + +import java.time.LocalDateTime; + +public record SocialVideoCandidate( + String externalVideoId, + String url, + String title, + String thumbnailUrl, + LocalDateTime publishedAt +) { } diff --git a/src/main/java/com/hlab/yanalyst/domain/social/SocialVideoRepository.java b/src/main/java/com/hlab/yanalyst/domain/social/SocialVideoRepository.java new file mode 100644 index 0000000..e5b343e --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/SocialVideoRepository.java @@ -0,0 +1,19 @@ +package com.hlab.yanalyst.domain.social; + +import com.hlab.yanalyst.domain.channel.FeedTopic; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.List; + +public interface SocialVideoRepository extends JpaRepository { + boolean existsByChannelIdAndExternalVideoId(Long channelId, String externalVideoId); + + @Query("select v from SocialVideo v join fetch v.channel c where c.topic = :topic " + + "order by coalesce(v.publishedAt, v.collectedAt) desc") + List findFeed(@Param("topic") FeedTopic topic, Pageable pageable); + + void deleteAllByChannelId(Long channelId); +} diff --git a/src/main/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClient.java b/src/main/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClient.java new file mode 100644 index 0000000..ef64de5 --- /dev/null +++ b/src/main/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClient.java @@ -0,0 +1,131 @@ +package com.hlab.yanalyst.domain.social; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** yt-dlp의 채널/사용자 목록 추출을 이용한 YouTube·TikTok·Instagram 수집기. */ +@Component +@RequiredArgsConstructor +public class YtDlpSocialFeedClient implements SocialFeedClient { + private final ObjectMapper objectMapper; + + @Value("${ytdlp.bin:yt-dlp}") private String ytdlpBin; + @Value("${hlab.social-feed.max-items:25}") private int maxItems; + @Value("${hlab.social-feed.timeout-seconds:90}") private long timeoutSeconds; + @Value("${hlab.social-feed.cookies-file:}") private String cookiesFile; + + @Override + public boolean supports(SocialPlatform platform) { + return platform != SocialPlatform.FACEBOOK; + } + + @Override + public SocialFeedBatch fetch(SocialChannel channel) { + List command = buildCommand(ytdlpBin, cookiesFile, maxItems, channel.getSourceUrl()); + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Thread drainer = Thread.ofVirtual().start(() -> { + try (var in = process.getInputStream()) { in.transferTo(output); } + catch (Exception ignored) { } + }); + if (!process.waitFor(Math.max(1, timeoutSeconds), TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IllegalStateException("소셜 채널 수집 제한 시간을 초과했습니다."); + } + drainer.join(2000); + String text = output.toString(StandardCharsets.UTF_8); + if (process.exitValue() != 0) throw new IllegalStateException(errorTail(text)); + return parse(text, channel.getPlatform(), Math.max(1, maxItems)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("소셜 채널 수집이 중단되었습니다.", e); + } catch (Exception e) { + if (e instanceof IllegalStateException state) throw state; + throw new IllegalStateException("yt-dlp로 채널을 읽지 못했습니다: " + e.getMessage(), e); + } + } + + static List buildCommand(String bin, String cookiesFile, int maxItems, String url) { + SocialPlatform.normalizeChannelUrl(url); + List command = new ArrayList<>(List.of(bin, "--flat-playlist", "--dump-single-json", + "--playlist-end", String.valueOf(Math.max(1, maxItems)), "--skip-download", "--no-warnings")); + if (cookiesFile != null && !cookiesFile.isBlank()) command.addAll(List.of("--cookies", cookiesFile.trim())); + command.add(url); + return command; + } + + SocialFeedBatch parse(String json, SocialPlatform platform, int limit) throws Exception { + JsonNode root = objectMapper.readTree(json); + String channelTitle = firstText(root, "channel", "uploader", "title"); + String channelId = firstText(root, "channel_id", "uploader_id", "id"); + List videos = new ArrayList<>(); + for (JsonNode entry : root.path("entries")) { + if (entry == null || entry.isNull()) continue; + String id = firstText(entry, "id", "display_id"); + if (id == null || id.isBlank()) continue; + String url = firstText(entry, "webpage_url", "url"); + if (url == null || !url.startsWith("http")) url = canonicalUrl(platform, id); + String title = firstText(entry, "title", "description"); + videos.add(new SocialVideoCandidate(id, url, + title == null || title.isBlank() ? "(제목 없음)" : title, + thumbnail(entry), publishedAt(entry))); + if (videos.size() >= limit) break; + } + return new SocialFeedBatch(channelTitle, channelId, videos); + } + + private static String firstText(JsonNode node, String... fields) { + for (String field : fields) { + String value = node.path(field).asText(null); + if (value != null && !value.isBlank()) return value; + } + return null; + } + + private static String thumbnail(JsonNode entry) { + String direct = entry.path("thumbnail").asText(null); + if (direct != null && !direct.isBlank()) return direct; + JsonNode thumbnails = entry.path("thumbnails"); + if (thumbnails.isArray() && !thumbnails.isEmpty()) { + return thumbnails.get(thumbnails.size() - 1).path("url").asText(null); + } + return null; + } + + private static LocalDateTime publishedAt(JsonNode entry) { + long timestamp = entry.path("timestamp").asLong(entry.path("release_timestamp").asLong(0)); + if (timestamp > 0) return LocalDateTime.ofInstant(Instant.ofEpochSecond(timestamp), ZoneId.systemDefault()); + String uploadDate = entry.path("upload_date").asText(null); + if (uploadDate != null && uploadDate.matches("\\d{8}")) { + return LocalDate.parse(uploadDate, DateTimeFormatter.BASIC_ISO_DATE).atStartOfDay(); + } + return null; + } + + private static String canonicalUrl(SocialPlatform platform, String id) { + return switch (platform) { + case YOUTUBE -> "https://www.youtube.com/watch?v=" + id; + case TIKTOK -> "https://www.tiktok.com/video/" + id; + case INSTAGRAM -> "https://www.instagram.com/p/" + id + "/"; + case FACEBOOK -> "https://www.facebook.com/reel/" + id; + }; + } + + private static String errorTail(String output) { + String clean = output == null ? "" : output.strip(); + if (clean.length() > 1200) clean = clean.substring(clean.length() - 1200); + return clean.isBlank() ? "yt-dlp가 채널 목록을 반환하지 않았습니다." : clean; + } +} diff --git a/src/main/java/com/hlab/yanalyst/web/WebController.java b/src/main/java/com/hlab/yanalyst/web/WebController.java index 7653f21..3c7b78b 100644 --- a/src/main/java/com/hlab/yanalyst/web/WebController.java +++ b/src/main/java/com/hlab/yanalyst/web/WebController.java @@ -74,6 +74,12 @@ public class WebController { return "feed"; } + @GetMapping("/distribution-feed") + public String distributionFeed(Model model) { + model.addAttribute("currentPage", "distribution-feed"); + return "distribution-feed"; + } + @GetMapping("/shortform") public String shortform(Model model) { model.addAttribute("currentPage", "shortform"); diff --git a/src/main/resources/application-local.yml.example b/src/main/resources/application-local.yml.example index 994753e..c8c7d9d 100644 --- a/src/main/resources/application-local.yml.example +++ b/src/main/resources/application-local.yml.example @@ -16,3 +16,6 @@ YOUTUBE_OAUTH_SETTINGS_FILE: config/youtube-oauth.properties # 페이스북 영상 다운로드 파일 저장 경로 (기본: downloads/facebook) # facebook-download: # dir: D:/downloads/facebook + +# Instagram/TikTok 채널이 로그인 쿠키를 요구할 때 yt-dlp Netscape 쿠키 파일 경로를 지정합니다. +# SOCIAL_FEED_COOKIES_FILE: D:/config/social-cookies.txt diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index fc73259..7a14866 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -129,6 +129,18 @@ hlab: # 생 조회수로 자르면 방금 올라온 영상까지 걸리므로 시간당 조회수로 거른다. min-views-per-hour: ${FEED_PERSON_MIN_VPH:20} + # 유통 소재용 소셜 채널 수집. Facebook 공개 페이지는 Page Plugin 피드, + # YouTube·TikTok·Instagram은 yt-dlp 채널 추출기를 사용한다. + social-feed: + enabled: ${SOCIAL_FEED_ENABLED:true} + cron: ${SOCIAL_FEED_CRON:0 */15 * * * *} + max-items: ${SOCIAL_FEED_MAX_ITEMS:25} + timeout-seconds: ${SOCIAL_FEED_TIMEOUT_SECONDS:90} + cookies-file: ${SOCIAL_FEED_COOKIES_FILE:} + seed: + phutunet-enabled: ${SOCIAL_SEED_PHUTUNET_ENABLED:true} + phutunet-url: ${SOCIAL_SEED_PHUTUNET_URL:https://www.facebook.com/phutunet/reels/} + # 텔레그램 알림: 아침 추천 다이제스트 + 피드 골든타임 소재(신규 수집분이 업로드 24h 이내일 때). # 토큰/챗ID 없으면 자동 no-op. notify: diff --git a/src/main/resources/templates/distribution-feed.html b/src/main/resources/templates/distribution-feed.html new file mode 100644 index 0000000..73dcca2 --- /dev/null +++ b/src/main/resources/templates/distribution-feed.html @@ -0,0 +1,67 @@ + + +h-lab - 유통 피드 + +
+
+
+

유통 피드

+

Facebook · Instagram · TikTok · YouTube 유통 채널의 새 영상을 모으고 텔레그램으로 알립니다.

+
+ +
+ + + + + + + +
+ + + + + + + + + + diff --git a/src/main/resources/templates/layout/sidebar.html b/src/main/resources/templates/layout/sidebar.html index 4a66776..6f4f479 100644 --- a/src/main/resources/templates/layout/sidebar.html +++ b/src/main/resources/templates/layout/sidebar.html @@ -27,6 +27,9 @@ 정치 피드 + + 유통 피드 + 숏폼 큐 diff --git a/src/test/java/com/hlab/yanalyst/domain/channel/FeedAlertServiceTest.java b/src/test/java/com/hlab/yanalyst/domain/channel/FeedAlertServiceTest.java new file mode 100644 index 0000000..4c19e34 --- /dev/null +++ b/src/test/java/com/hlab/yanalyst/domain/channel/FeedAlertServiceTest.java @@ -0,0 +1,51 @@ +package com.hlab.yanalyst.domain.channel; + +import com.hlab.yanalyst.global.notify.TelegramNotifier; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +class FeedAlertServiceTest { + + @Test + void excludesPoliticsButKeepsEntertainmentAlerts() { + TelegramNotifier notifier = mock(TelegramNotifier.class); + when(notifier.isEnabled()).thenReturn(true); + when(notifier.sendMessage(anyString())).thenReturn(true); + FeedAlertService service = new FeedAlertService(notifier); + ChannelVideo entertainment = video("entertainment-id", FeedTopic.ENTERTAINMENT); + ChannelVideo politics = video("politics-video-id", FeedTopic.POLITICS); + + int alerted = service.notifyGoldenTime("소재 피드", List.of(politics, entertainment)); + + assertThat(alerted).isEqualTo(1); + verify(notifier).sendMessage(argThat(message -> message.contains("entertainment-id") + && !message.contains("politics-video-id"))); + } + + @Test + void skipsMessageWhenOnlyPoliticsVideosAreNew() { + TelegramNotifier notifier = mock(TelegramNotifier.class); + when(notifier.isEnabled()).thenReturn(true); + FeedAlertService service = new FeedAlertService(notifier); + + assertThat(service.notifyGoldenTime("정치 피드", List.of(video("politics-video-id", FeedTopic.POLITICS)))) + .isZero(); + verify(notifier, never()).sendMessage(anyString()); + } + + private static ChannelVideo video(String id, FeedTopic topic) { + ChannelVideo video = ChannelVideo.builder() + .videoId(id) + .title(id) + .publishedAt(LocalDateTime.now().minusMinutes(5)) + .build(); + video.applyFeedInfo("channel", "채널", 1L, null, ChannelRole.SOURCE, topic); + return video; + } +} diff --git a/src/test/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleServiceTest.java b/src/test/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleServiceTest.java index d6641f3..a2b568e 100644 --- a/src/test/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleServiceTest.java +++ b/src/test/java/com/hlab/yanalyst/domain/shortform/ShortformSubtitleServiceTest.java @@ -11,6 +11,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.times; @@ -78,4 +79,23 @@ class ShortformSubtitleServiceTest { assertThat(job.getSubtitleVtt()).isEqualTo("WEBVTT\n"); verify(telegramNotifier).sendMessage(org.mockito.ArgumentMatchers.contains("자막 생성 완료")); } + + @Test + void doesNotNotifyForPoliticsJob() throws Exception { + ShortformJobRepository repository = mock(ShortformJobRepository.class); + ShortformJob job = ShortformJob.create( + "https://www.youtube.com/watch?v=gDfGqRDubCg", + "gDfGqRDubCg", + ShortformCategory.POLITICS); + when(repository.findById(1L)).thenReturn(Optional.of(job)); + TelegramNotifier telegramNotifier = mock(TelegramNotifier.class); + ShortformSubtitleService service = spy(new ShortformSubtitleService( + repository, mock(ShortformWhisperService.class), telegramNotifier)); + doReturn("WEBVTT\n").when(service).collectOrTranscribe(1L, "gDfGqRDubCg"); + + service.collectAsync(1L); + + assertThat(job.getSubtitleStatus()).isEqualTo(SubtitleStatus.READY); + verify(telegramNotifier, never()).sendMessage(org.mockito.ArgumentMatchers.anyString()); + } } diff --git a/src/test/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClientTest.java b/src/test/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClientTest.java new file mode 100644 index 0000000..352943e --- /dev/null +++ b/src/test/java/com/hlab/yanalyst/domain/social/FacebookPageFeedClientTest.java @@ -0,0 +1,35 @@ +package com.hlab.yanalyst.domain.social; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class FacebookPageFeedClientTest { + + @Test + void parsesUniqueReelsWithTitleAndPublishedTime() { + String markup = """ +
+
Phutu Crafts
+ 15시간 전 +

How to Fix a Broken Outlet

+ + 댓글 +
+
+ 6일 +
An Outdoor Light Hidden Inside a Tree
+
+ """; + + SocialFeedBatch batch = FacebookPageFeedClient.parseMarkup(markup, 25); + + assertThat(batch.channelTitle()).isEqualTo("Phutu Crafts"); + assertThat(batch.videos()).hasSize(2); + assertThat(batch.videos().get(0).externalVideoId()).isEqualTo("2234599224051571"); + assertThat(batch.videos().get(0).url()).isEqualTo("https://www.facebook.com/reel/2234599224051571"); + assertThat(batch.videos().get(0).title()).isEqualTo("How to Fix a Broken Outlet"); + assertThat(batch.videos().get(0).thumbnailUrl()).isEqualTo("https://cdn.example/thumb.jpg"); + assertThat(batch.videos().get(0).publishedAt()).isNotNull(); + } +} diff --git a/src/test/java/com/hlab/yanalyst/domain/social/SocialFeedServiceTest.java b/src/test/java/com/hlab/yanalyst/domain/social/SocialFeedServiceTest.java new file mode 100644 index 0000000..64264a1 --- /dev/null +++ b/src/test/java/com/hlab/yanalyst/domain/social/SocialFeedServiceTest.java @@ -0,0 +1,48 @@ +package com.hlab.yanalyst.domain.social; + +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class SocialFeedServiceTest { + + @Test + void firstCollectionCreatesBaselineAndOnlyLaterVideosAlert() { + SocialChannelRepository channels = mock(SocialChannelRepository.class); + SocialVideoRepository videos = mock(SocialVideoRepository.class); + SocialFeedClient client = mock(SocialFeedClient.class); + SocialFeedAlertService alerts = mock(SocialFeedAlertService.class); + SocialChannel channel = SocialChannel.register("https://facebook.com/phutunet/reels/", "DISTRIBUTION", true); + SocialVideoCandidate existing = candidate("old"); + SocialVideoCandidate newlyPublished = candidate("new"); + + when(channels.findById(1L)).thenReturn(java.util.Optional.of(channel)); + when(client.supports(SocialPlatform.FACEBOOK)).thenReturn(true); + when(client.fetch(channel)).thenReturn( + new SocialFeedBatch("Phutu Crafts", null, List.of(existing)), + new SocialFeedBatch("Phutu Crafts", null, List.of(newlyPublished))); + when(videos.existsByChannelIdAndExternalVideoId(eq(1L), anyString())).thenReturn(false); + when(videos.save(any(SocialVideo.class))).thenAnswer(invocation -> invocation.getArgument(0)); + when(alerts.notifyNewVideos(eq(channel), anyList())).thenReturn(1); + SocialFeedService service = new SocialFeedService(channels, videos, List.of(client), alerts); + + SocialFeedService.CollectionResult baseline = service.collectOne(1L); + SocialFeedService.CollectionResult next = service.collectOne(1L); + + assertThat(baseline.baselineCollection()).isTrue(); + assertThat(baseline.alerted()).isZero(); + assertThat(next.baselineCollection()).isFalse(); + assertThat(next.alerted()).isEqualTo(1); + verify(alerts, times(1)).notifyNewVideos(eq(channel), anyList()); + } + + private static SocialVideoCandidate candidate(String id) { + return new SocialVideoCandidate(id, "https://facebook.com/reel/" + id, id, null, + LocalDateTime.now()); + } +} diff --git a/src/test/java/com/hlab/yanalyst/domain/social/SocialPlatformTest.java b/src/test/java/com/hlab/yanalyst/domain/social/SocialPlatformTest.java new file mode 100644 index 0000000..e011322 --- /dev/null +++ b/src/test/java/com/hlab/yanalyst/domain/social/SocialPlatformTest.java @@ -0,0 +1,35 @@ +package com.hlab.yanalyst.domain.social; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.*; + +class SocialPlatformTest { + + @Test + void detectsAndNormalizesSupportedChannelUrls() { + assertThat(SocialPlatform.fromUrl("https://www.facebook.com/phutunet/reels/")) + .isEqualTo(SocialPlatform.FACEBOOK); + assertThat(SocialPlatform.fromUrl("https://www.instagram.com/shop/")) + .isEqualTo(SocialPlatform.INSTAGRAM); + assertThat(SocialPlatform.fromUrl("https://www.tiktok.com/@shop")) + .isEqualTo(SocialPlatform.TIKTOK); + assertThat(SocialPlatform.fromUrl("https://youtube.com/@shop")) + .isEqualTo(SocialPlatform.YOUTUBE); + + assertThat(SocialPlatform.normalizeChannelUrl("http://WWW.FACEBOOK.COM/phutunet/reels/?ref=home")) + .isEqualTo("https://www.facebook.com/phutunet"); + } + + @Test + void rejectsUnknownHostsCredentialsAndIndividualVideos() { + assertThatThrownBy(() -> SocialPlatform.fromUrl("https://example.com/channel")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SocialPlatform.fromUrl("https://user@facebook.com/page")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SocialPlatform.normalizeChannelUrl("https://facebook.com/reel/1234")) + .hasMessageContaining("개별 영상"); + assertThatThrownBy(() -> SocialPlatform.normalizeChannelUrl("https://youtube.com/watch?v=1234")) + .hasMessageContaining("개별 영상"); + } +} diff --git a/src/test/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClientTest.java b/src/test/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClientTest.java new file mode 100644 index 0000000..5b8b63f --- /dev/null +++ b/src/test/java/com/hlab/yanalyst/domain/social/YtDlpSocialFeedClientTest.java @@ -0,0 +1,30 @@ +package com.hlab.yanalyst.domain.social; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class YtDlpSocialFeedClientTest { + + @Test + void parsesFlatPlaylistJson() throws Exception { + String json = """ + {"channel":"유통 채널","channel_id":"UC-shop","entries":[ + {"id":"abcdefghijk","title":"새 상품 영상","url":"abcdefghijk","timestamp":1788448574, + "thumbnails":[{"url":"https://cdn.example/small.jpg"},{"url":"https://cdn.example/large.jpg"}]} + ]} + """; + YtDlpSocialFeedClient client = new YtDlpSocialFeedClient(new ObjectMapper()); + + SocialFeedBatch batch = client.parse(json, SocialPlatform.YOUTUBE, 25); + + assertThat(batch.channelTitle()).isEqualTo("유통 채널"); + assertThat(batch.externalChannelId()).isEqualTo("UC-shop"); + assertThat(batch.videos()).singleElement().satisfies(video -> { + assertThat(video.url()).isEqualTo("https://www.youtube.com/watch?v=abcdefghijk"); + assertThat(video.thumbnailUrl()).isEqualTo("https://cdn.example/large.jpg"); + assertThat(video.publishedAt()).isNotNull(); + }); + } +}